KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): modularize getFfmpegOptions (#3138)

Browse Source 
* refactored `getFfmpegOptions`

refactor transcoding, make separate service

* fixed enum casing

* use `Logger` instead of `console.log`

* review suggestions

* use enum for `getHandler`

* fixed formatting

* Update server/src/domain/media/media.util.ts

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

* Update server/src/domain/media/media.util.ts

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

* More specific imports, renamed codec classes

* simplified code

* removed unused import

* added tests

* added base implementation for bitrate and threads

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Mert
2023-07-08 22:43:11 -04:00
committed by GitHub
parent 27018e4ab6
commit 8349a28ed8
33 changed files with 1131 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
server/immich-openapi-specs.json
View File

@@ -4929,6 +4929,14 @@
"OTHER"
]
},
"AudioCodec": {
"type": "string",
"enum": [
"mp3",
"aac",
"opus"
]
},
"AuthDeviceResponseDto": {
"type": "object",
"properties": {
@@ -6347,13 +6355,16 @@
"threads": {
"type": "integer"
},
"preset": {
"type": "string"
},
"targetVideoCodec": {
"type": "string"
"$ref": "#/components/schemas/VideoCodec"
},
"targetAudioCodec": {
"$ref": "#/components/schemas/AudioCodec"
},
"transcode": {
"$ref": "#/components/schemas/TranscodePolicy"
},
"preset": {
"type": "string"
},
"targetResolution": {
@@ -6364,27 +6375,18 @@
},
"twoPass": {
"type": "boolean"
},
"transcode": {
"type": "string",
"enum": [
"all",
"optimal",
"required",
"disabled"
]
}
},
"required": [
"crf",
"threads",
"preset",
"targetVideoCodec",
"targetAudioCodec",
"transcode",
"preset",
"targetResolution",
"maxBitrate",
"twoPass",
"transcode"
"twoPass"
]
},
"SystemConfigJobDto": {
@@ -6604,6 +6606,15 @@
"month"
]
},
"TranscodePolicy": {
"type": "string",
"enum": [
"all",
"optimal",
"required",
"disabled"
]
},
"UpdateAlbumDto": {
"type": "object",
"properties": {
@@ -6804,6 +6815,14 @@
"required": [
"authStatus"
]
},
"VideoCodec": {
"type": "string",
"enum": [
"h264",
"hevc",
"vp9"
]
}
}
}

12
server/src/domain/media/media.repository.ts
View File

@@ -39,10 +39,22 @@ export interface CropOptions {
}
export interface TranscodeOptions {
inputOptions: string[];
outputOptions: string[];
twoPass: boolean;
}
export interface BitrateDistribution {
max: number;
target: number;
min: number;
unit: string;
}
export interface VideoCodecSWConfig {
getOptions(stream: VideoStreamInfo): TranscodeOptions;
}
export interface IMediaRepository {
// image
resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;

232
server/src/domain/media/media.service.spec.ts
View File

@@ -1,4 +1,4 @@
import { AssetType, SystemConfigKey } from '@app/infra/entities';
import { AssetType, SystemConfigKey, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import {
assetEntityStub,
newAssetRepositoryMock,
@@ -104,6 +104,13 @@ describe(MediaService.name, () => {
});
describe('handleGenerateJpegThumbnail', () => {
it('should skip thumbnail generation if asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
await sut.handleGenerateJpegThumbnail({ id: assetEntityStub.image.id });
expect(mediaMock.resize).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalledWith();
});
it('should generate a thumbnail for an image', async () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
await sut.handleGenerateJpegThumbnail({ id: assetEntityStub.image.id });
@@ -142,15 +149,22 @@ describe(MediaService.name, () => {
});
describe('handleGenerateWebpThumbnail', () => {
it('should skip thumbnail generation if asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
await sut.handleGenerateWebpThumbnail({ id: assetEntityStub.image.id });
expect(mediaMock.resize).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalledWith();
});
it('should skip thumbnail generate if resize path is missing', async () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]);
await sut.handleGenerateWepbThumbnail({ id: assetEntityStub.noResizePath.id });
await sut.handleGenerateWebpThumbnail({ id: assetEntityStub.noResizePath.id });
expect(mediaMock.resize).not.toHaveBeenCalled();
});
it('should generate a thumbnail', async () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
await sut.handleGenerateWepbThumbnail({ id: assetEntityStub.image.id });
await sut.handleGenerateWebpThumbnail({ id: assetEntityStub.image.id });
expect(mediaMock.resize).toHaveBeenCalledWith(
'/uploads/user-id/thumbs/path.ext',
@@ -162,6 +176,12 @@ describe(MediaService.name, () => {
});
describe('handleGenerateThumbhashThumbnail', () => {
it('should skip thumbhash generation if asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.image.id });
expect(mediaMock.generateThumbhash).not.toHaveBeenCalled();
});
it('should skip thumbhash generation if resize path is missing', async () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]);
await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.noResizePath.id });
@@ -219,6 +239,20 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
});
it('should skip transcoding if asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.probe).not.toHaveBeenCalled();
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should skip transcoding if non-video asset', async () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
await sut.handleVideoConversion({ id: assetEntityStub.image.id });
expect(mediaMock.probe).not.toHaveBeenCalled();
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should transcode the longest stream', async () => {
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
mediaMock.probe.mockResolvedValue(probeStub.multipleVideoStreams);
@@ -232,6 +266,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -261,13 +296,14 @@ describe(MediaService.name, () => {
it('should transcode when set to all', async () => {
mediaMock.probe.mockResolvedValue(probeStub.multipleVideoStreams);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'all' }]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.ALL }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -283,12 +319,13 @@ describe(MediaService.name, () => {
it('should transcode when optimal and too big', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -306,7 +343,7 @@ describe(MediaService.name, () => {
it('should not scale resolution if no target resolution', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'all' },
{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.ALL },
{ key: SystemConfigKey.FFMPEG_TARGET_RESOLUTION, value: 'original' },
]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
@@ -314,6 +351,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -329,13 +367,14 @@ describe(MediaService.name, () => {
it('should transcode with alternate scaling video is vertical', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -352,13 +391,14 @@ describe(MediaService.name, () => {
it('should transcode when audio doesnt match target', async () => {
mediaMock.probe.mockResolvedValue(probeStub.audioStreamMp3);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -375,13 +415,14 @@ describe(MediaService.name, () => {
it('should transcode when container doesnt match target', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -404,6 +445,22 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should not transcode if transcoding is disabled', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.DISABLED }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should not transcode if target codec is invalid', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: 'invalid' }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should set max bitrate if above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '4500k' }]);
@@ -413,6 +470,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -441,6 +499,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -466,6 +525,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -480,11 +540,12 @@ describe(MediaService.name, () => {
);
});
it('should configure preset for vp9', async () => {
it('should transcode by bitrate in two passes for vp9 when two pass mode and max bitrate are enabled', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: 'vp9' },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '4500k' },
{ key: SystemConfigKey.FFMPEG_TWO_PASS, value: true },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
@@ -492,6 +553,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec vp9',
'-acodec aac',
@@ -500,7 +562,64 @@ describe(MediaService.name, () => {
'-vf scale=-2:720',
'-cpu-used 5',
'-row-mt 1',
'-threads 2',
'-b:v 3104k',
'-minrate 1552k',
'-maxrate 4500k',
],
twoPass: true,
},
);
});
it('should configure preset for vp9', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'slow' },
]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec vp9',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-vf scale=-2:720',
'-cpu-used 2',
'-row-mt 1',
'-crf 23',
'-b:v 0',
],
twoPass: false,
},
);
});
it('should not configure preset for vp9 if invalid', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec vp9',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-vf scale=-2:720',
'-row-mt 1',
'-crf 23',
'-b:v 0',
],
@@ -512,7 +631,7 @@ describe(MediaService.name, () => {
it('should configure threads if above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: 'vp9' },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 },
]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
@@ -521,6 +640,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec vp9',
'-acodec aac',
@@ -538,7 +658,7 @@ describe(MediaService.name, () => {
);
});
it('should disable thread pooling for x264/x265 if thread limit is above 0', async () => {
it('should disable thread pooling for h264 if thread limit is above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
@@ -547,6 +667,7 @@ describe(MediaService.name, () => {
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
@@ -563,5 +684,86 @@ describe(MediaService.name, () => {
},
);
});
it('should omit thread flags for h264 if thread limit is at or below 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_THREADS, value: 0 }]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
it('should disable thread pooling for hevc if thread limit is above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec hevc',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-vf scale=-2:720',
'-preset ultrafast',
'-threads 2',
'-x265-params "pools=none"',
'-x265-params "frame-threads=2"',
'-crf 23',
],
twoPass: false,
},
);
});
it('should omit thread flags for hevc if thread limit is at or below 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_THREADS, value: 0 },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
]);
assetMock.getByIds.mockResolvedValue([assetEntityStub.video]);
await sut.handleVideoConversion({ id: assetEntityStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec hevc',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
});
});

131
server/src/domain/media/media.service.ts
View File

@@ -1,5 +1,5 @@
import { AssetEntity, AssetType, TranscodePreset } from '@app/infra/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { AssetEntity, AssetType, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common';
import { join } from 'path';
import { IAssetRepository, WithoutProperty } from '../asset';
import { usePagination } from '../domain.util';
@@ -9,6 +9,7 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config
import { SystemConfigCore } from '../system-config/system-config.core';
import { JPEG_THUMBNAIL_SIZE, WEBP_THUMBNAIL_SIZE } from './media.constant';
import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository';
import { H264Config, HEVCConfig, VP9Config } from './media.util';
@Injectable()
export class MediaService {
@@ -82,7 +83,7 @@ export class MediaService {
return true;
}
async handleGenerateWepbThumbnail({ id }: IEntityJob) {
async handleGenerateWebpThumbnail({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset || !asset.resizePath) {
return false;
@@ -152,11 +153,16 @@ export class MediaService {
return false;
}
const outputOptions = this.getFfmpegOptions(mainVideoStream, config);
const twoPass = this.eligibleForTwoPass(config);
let transcodeOptions;
try {
transcodeOptions = this.getCodecConfig(config).getOptions(mainVideoStream);
} catch (err) {
this.logger.error(`An error occurred while configuring transcoding options: ${err}`);
return false;
}
this.logger.log(`Start encoding video ${asset.id} ${outputOptions}`);
await this.mediaRepository.transcode(input, output, { outputOptions, twoPass });
this.logger.log(`Start encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`);
await this.mediaRepository.transcode(input, output, transcodeOptions);
this.logger.log(`Encoding success ${asset.id}`);
@@ -199,16 +205,16 @@ export class MediaService {
const isLargerThanTargetRes = scalingEnabled && Math.min(videoStream.height, videoStream.width) > targetRes;
switch (ffmpegConfig.transcode) {
case TranscodePreset.DISABLED:
case TranscodePolicy.DISABLED:
return false;
case TranscodePreset.ALL:
case TranscodePolicy.ALL:
return true;
case TranscodePreset.REQUIRED:
case TranscodePolicy.REQUIRED:
return !allTargetsMatching;
case TranscodePreset.OPTIMAL:
case TranscodePolicy.OPTIMAL:
return !allTargetsMatching || isLargerThanTargetRes;
default:
@@ -216,99 +222,16 @@ export class MediaService {
}
}
private getFfmpegOptions(stream: VideoStreamInfo, ffmpeg: SystemConfigFFmpegDto) {
const options = [
`-vcodec ${ffmpeg.targetVideoCodec}`,
`-acodec ${ffmpeg.targetAudioCodec}`,
// Makes a second pass moving the moov atom to the beginning of
// the file for improved playback speed.
'-movflags faststart',
'-fps_mode passthrough',
];
// video dimensions
const videoIsRotated = Math.abs(stream.rotation) === 90;
const scalingEnabled = ffmpeg.targetResolution !== 'original';
const targetResolution = Number.parseInt(ffmpeg.targetResolution);
const isVideoVertical = stream.height > stream.width || videoIsRotated;
const scaling = isVideoVertical ? `${targetResolution}:-2` : `-2:${targetResolution}`;
const shouldScale = scalingEnabled && Math.min(stream.height, stream.width) > targetResolution;
// video codec
const isVP9 = ffmpeg.targetVideoCodec === 'vp9';
const isH264 = ffmpeg.targetVideoCodec === 'h264';
const isH265 = ffmpeg.targetVideoCodec === 'hevc';
// transcode efficiency
const limitThreads = ffmpeg.threads > 0;
const maxBitrateValue = Number.parseInt(ffmpeg.maxBitrate) || 0;
const constrainMaximumBitrate = maxBitrateValue > 0;
const bitrateUnit = ffmpeg.maxBitrate.trim().substring(maxBitrateValue.toString().length); // use inputted unit if provided
if (shouldScale) {
options.push(`-vf scale=${scaling}`);
private getCodecConfig(config: SystemConfigFFmpegDto) {
switch (config.targetVideoCodec) {
case VideoCodec.H264:
return new H264Config(config);
case VideoCodec.HEVC:
return new HEVCConfig(config);
case VideoCodec.VP9:
return new VP9Config(config);
default:
throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
}
if (isH264 || isH265) {
options.push(`-preset ${ffmpeg.preset}`);
}
if (isVP9) {
// vp9 doesn't have presets, but does have a similar setting -cpu-used, from 0-5, 0 being the slowest
const presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
const speed = Math.min(presets.indexOf(ffmpeg.preset), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
if (speed >= 0) {
options.push(`-cpu-used ${speed}`);
}
options.push('-row-mt 1'); // better multithreading
}
if (limitThreads) {
options.push(`-threads ${ffmpeg.threads}`);
// x264 and x265 handle threads differently than one might expect
// https://x265.readthedocs.io/en/latest/cli.html#cmdoption-pools
if (isH264 || isH265) {
options.push(`-${isH265 ? 'x265' : 'x264'}-params "pools=none"`);
options.push(`-${isH265 ? 'x265' : 'x264'}-params "frame-threads=${ffmpeg.threads}"`);
}
}
// two-pass mode for x264/x265 uses bitrate ranges, so it requires a max bitrate from which to derive a target and min bitrate
if (constrainMaximumBitrate && ffmpeg.twoPass) {
const targetBitrateValue = Math.ceil(maxBitrateValue / 1.45); // recommended by https://developers.google.com/media/vp9/settings/vod
const minBitrateValue = targetBitrateValue / 2;
options.push(`-b:v ${targetBitrateValue}${bitrateUnit}`);
options.push(`-minrate ${minBitrateValue}${bitrateUnit}`);
options.push(`-maxrate ${maxBitrateValue}${bitrateUnit}`);
} else if (constrainMaximumBitrate || isVP9) {
// for vp9, these flags work for both one-pass and two-pass
options.push(`-crf ${ffmpeg.crf}`);
if (isVP9) {
options.push(`-b:v ${maxBitrateValue}${bitrateUnit}`);
} else {
options.push(`-maxrate ${maxBitrateValue}${bitrateUnit}`);
// -bufsize is the peak possible bitrate at any moment, while -maxrate is the max rolling average bitrate
// needed for -maxrate to be enforced
options.push(`-bufsize ${maxBitrateValue * 2}${bitrateUnit}`);
}
} else {
options.push(`-crf ${ffmpeg.crf}`);
}
return options;
}
private eligibleForTwoPass(ffmpeg: SystemConfigFFmpegDto) {
if (!ffmpeg.twoPass) {
return false;
}
const isVP9 = ffmpeg.targetVideoCodec === 'vp9';
const maxBitrateValue = Number.parseInt(ffmpeg.maxBitrate) || 0;
const constrainMaximumBitrate = maxBitrateValue > 0;
return constrainMaximumBitrate || isVP9;
}
}

191
server/src/domain/media/media.util.ts Normal file
View File

@@ -0,0 +1,191 @@
import { SystemConfigFFmpegDto } from '../system-config/dto';
import { BitrateDistribution, TranscodeOptions, VideoCodecSWConfig, VideoStreamInfo } from './media.repository';
class BaseConfig implements VideoCodecSWConfig {
constructor(protected config: SystemConfigFFmpegDto) {}
getOptions(stream: VideoStreamInfo) {
const options = {
inputOptions: this.getBaseInputOptions(),
outputOptions: this.getBaseOutputOptions(),
twoPass: this.eligibleForTwoPass(),
} as TranscodeOptions;
const filters = this.getFilterOptions(stream);
if (filters.length > 0) {
options.outputOptions.push(`-vf ${filters.join(',')}`);
}
options.outputOptions.push(...this.getPresetOptions());
options.outputOptions.push(...this.getThreadOptions());
options.outputOptions.push(...this.getBitrateOptions());
return options;
}
getBaseInputOptions(): string[] {
return [];
}
getBaseOutputOptions() {
return [
`-vcodec ${this.config.targetVideoCodec}`,
`-acodec ${this.config.targetAudioCodec}`,
// Makes a second pass moving the moov atom to the beginning of
// the file for improved playback speed.
'-movflags faststart',
'-fps_mode passthrough',
];
}
getFilterOptions(stream: VideoStreamInfo) {
const options = [];
if (this.shouldScale(stream)) {
options.push(`scale=${this.getScaling(stream)}`);
}
return options;
}
getPresetOptions() {
return [`-preset ${this.config.preset}`];
}
getBitrateOptions() {
const bitrates = this.getBitrateDistribution();
if (this.eligibleForTwoPass()) {
return [
`-b:v ${bitrates.target}${bitrates.unit}`,
`-minrate ${bitrates.min}${bitrates.unit}`,
`-maxrate ${bitrates.max}${bitrates.unit}`,
];
} else if (bitrates.max > 0) {
// -bufsize is the peak possible bitrate at any moment, while -maxrate is the max rolling average bitrate
return [
`-crf ${this.config.crf}`,
`-maxrate ${bitrates.max}${bitrates.unit}`,
`-bufsize ${bitrates.max * 2}${bitrates.unit}`,
];
} else {
return [`-crf ${this.config.crf}`];
}
}
getThreadOptions(): Array<string> {
if (this.config.threads <= 0) {
return [];
}
return [`-threads ${this.config.threads}`];
}
eligibleForTwoPass() {
if (!this.config.twoPass) {
return false;
}
return this.isBitrateConstrained() || this.config.targetVideoCodec === 'vp9';
}
getBitrateDistribution() {
const max = this.getMaxBitrateValue();
const target = Math.ceil(max / 1.45); // recommended by https://developers.google.com/media/vp9/settings/vod
const min = target / 2;
const unit = this.getBitrateUnit();
return { max, target, min, unit } as BitrateDistribution;
}
getTargetResolution(stream: VideoStreamInfo) {
if (this.config.targetResolution === 'original') {
return Math.min(stream.height, stream.width);
}
return Number.parseInt(this.config.targetResolution);
}
shouldScale(stream: VideoStreamInfo) {
return Math.min(stream.height, stream.width) > this.getTargetResolution(stream);
}
getScaling(stream: VideoStreamInfo) {
const targetResolution = this.getTargetResolution(stream);
return this.isVideoVertical(stream) ? `${targetResolution}:-2` : `-2:${targetResolution}`;
}
isVideoRotated(stream: VideoStreamInfo) {
return Math.abs(stream.rotation) === 90;
}
isVideoVertical(stream: VideoStreamInfo) {
return stream.height > stream.width || this.isVideoRotated(stream);
}
isBitrateConstrained() {
return this.getMaxBitrateValue() > 0;
}
getBitrateUnit() {
const maxBitrate = this.getMaxBitrateValue();
return this.config.maxBitrate.trim().substring(maxBitrate.toString().length); // use inputted unit if provided
}
getMaxBitrateValue() {
return Number.parseInt(this.config.maxBitrate) || 0;
}
getPresetIndex() {
const presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
return presets.indexOf(this.config.preset);
}
}
export class H264Config extends BaseConfig {
getThreadOptions() {
if (this.config.threads <= 0) {
return [];
}
return [
...super.getThreadOptions(),
'-x264-params "pools=none"',
`-x264-params "frame-threads=${this.config.threads}"`,
];
}
}
export class HEVCConfig extends BaseConfig {
getThreadOptions() {
if (this.config.threads <= 0) {
return [];
}
return [
...super.getThreadOptions(),
'-x265-params "pools=none"',
`-x265-params "frame-threads=${this.config.threads}"`,
];
}
}
export class VP9Config extends BaseConfig {
getPresetOptions() {
const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
if (speed >= 0) {
return [`-cpu-used ${speed}`];
}
return [];
}
getBitrateOptions() {
const bitrates = this.getBitrateDistribution();
if (this.eligibleForTwoPass()) {
return [
`-b:v ${bitrates.target}${bitrates.unit}`,
`-minrate ${bitrates.min}${bitrates.unit}`,
`-maxrate ${bitrates.max}${bitrates.unit}`,
];
}
return [`-crf ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`];
}
getThreadOptions() {
return ['-row-mt 1', ...super.getThreadOptions()];
}
}

17
server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts
View File

@@ -1,4 +1,4 @@
import { TranscodePreset } from '@app/infra/entities';
import { AudioCodec, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsBoolean, IsEnum, IsInt, IsString, Max, Min } from 'class-validator';
@@ -20,11 +20,13 @@ export class SystemConfigFFmpegDto {
@IsString()
preset!: string;
@IsString()
targetVideoCodec!: string;
@IsEnum(VideoCodec)
@ApiProperty({ enumName: 'VideoCodec', enum: VideoCodec })
targetVideoCodec!: VideoCodec;
@IsString()
targetAudioCodec!: string;
@IsEnum(AudioCodec)
@ApiProperty({ enumName: 'AudioCodec', enum: AudioCodec })
targetAudioCodec!: AudioCodec;
@IsString()
targetResolution!: string;
@@ -35,6 +37,7 @@ export class SystemConfigFFmpegDto {
@IsBoolean()
twoPass!: boolean;
@IsEnum(TranscodePreset)
transcode!: TranscodePreset;
@IsEnum(TranscodePolicy)
@ApiProperty({ enumName: 'TranscodePolicy', enum: TranscodePolicy })
transcode!: TranscodePolicy;
}

10
server/src/domain/system-config/system-config.core.ts
View File

@@ -1,9 +1,11 @@
import {
AudioCodec,
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
SystemConfigValue,
TranscodePreset,
TranscodePolicy,
VideoCodec,
} from '@app/infra/entities';
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import * as _ from 'lodash';
@@ -19,12 +21,12 @@ const defaults = Object.freeze<SystemConfig>({
crf: 23,
threads: 0,
preset: 'ultrafast',
targetVideoCodec: 'h264',
targetAudioCodec: 'aac',
targetVideoCodec: VideoCodec.H264,
targetAudioCodec: AudioCodec.AAC,
targetResolution: '720',
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
transcode: TranscodePolicy.REQUIRED,
},
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },

15
server/src/domain/system-config/system-config.service.spec.ts
View File

@@ -1,4 +1,11 @@
import { SystemConfig, SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities';
import {
AudioCodec,
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
TranscodePolicy,
VideoCodec,
} from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import { newJobRepositoryMock, newSystemConfigRepositoryMock, systemConfigStub } from '@test';
import { IJobRepository, JobName, QueueName } from '../job';
@@ -28,12 +35,12 @@ const updatedConfig = Object.freeze<SystemConfig>({
crf: 30,
threads: 0,
preset: 'ultrafast',
targetAudioCodec: 'aac',
targetAudioCodec: AudioCodec.AAC,
targetResolution: '720',
targetVideoCodec: 'h264',
targetVideoCodec: VideoCodec.H264,
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
transcode: TranscodePolicy.REQUIRED,
},
oauth: {
autoLaunch: true,

20
server/src/infra/entities/system-config.entity.ts
View File

@@ -51,24 +51,36 @@ export enum SystemConfigKey {
STORAGE_TEMPLATE = 'storageTemplate.template',
}
export enum TranscodePreset {
export enum TranscodePolicy {
ALL = 'all',
OPTIMAL = 'optimal',
REQUIRED = 'required',
DISABLED = 'disabled',
}
export enum VideoCodec {
H264 = 'h264',
HEVC = 'hevc',
VP9 = 'vp9',
}
export enum AudioCodec {
MP3 = 'mp3',
AAC = 'aac',
OPUS = 'opus',
}
export interface SystemConfig {
ffmpeg: {
crf: number;
threads: number;
preset: string;
targetVideoCodec: string;
targetAudioCodec: string;
targetVideoCodec: VideoCodec;
targetAudioCodec: AudioCodec;
targetResolution: string;
maxBitrate: string;
twoPass: boolean;
transcode: TranscodePreset;
transcode: TranscodePolicy;
};
job: Record<QueueName, { concurrency: number }>;
oauth: {

2
server/src/infra/infra.module.ts
View File

@@ -65,7 +65,6 @@ const providers: Provider[] = [
{ provide: IJobRepository, useClass: JobRepository },
{ provide: IKeyRepository, useClass: APIKeyRepository },
{ provide: IMachineLearningRepository, useClass: MachineLearningRepository },
{ provide: IMediaRepository, useClass: MediaRepository },
{ provide: IPartnerRepository, useClass: PartnerRepository },
{ provide: IPersonRepository, useClass: PersonRepository },
{ provide: ISearchRepository, useClass: TypesenseRepository },
@@ -74,6 +73,7 @@ const providers: Provider[] = [
{ provide: IStorageRepository, useClass: FilesystemProvider },
{ provide: ISystemConfigRepository, useClass: SystemConfigRepository },
{ provide: ITagRepository, useClass: TagRepository },
{ provide: IMediaRepository, useClass: MediaRepository },
{ provide: IUserRepository, useClass: UserRepository },
{ provide: IUserTokenRepository, useClass: UserTokenRepository },
];

23
server/src/infra/repositories/media.repository.ts
View File

@@ -1,4 +1,5 @@
import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain';
import { Logger } from '@nestjs/common';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import fs from 'fs/promises';
import sharp from 'sharp';
@@ -7,6 +8,8 @@ import { promisify } from 'util';
const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
export class MediaRepository implements IMediaRepository {
private logger = new Logger(MediaRepository.name);
crop(input: string, options: CropOptions): Promise<Buffer> {
return sharp(input, { failOnError: false })
.extract({
@@ -47,7 +50,10 @@ export class MediaRepository implements IMediaRepository {
`-vf scale='min(${size},iw)':'min(${size},ih)':force_original_aspect_ratio=increase`,
])
.output(output)
.on('error', reject)
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('end', resolve)
.run();
});
@@ -87,7 +93,10 @@ export class MediaRepository implements IMediaRepository {
ffmpeg(input, { niceness: 10 })
.outputOptions(options.outputOptions)
.output(output)
.on('error', reject)
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('end', resolve)
.run();
});
@@ -102,7 +111,10 @@ export class MediaRepository implements IMediaRepository {
.addOptions('-passlogfile', output)
.addOptions('-f null')
.output('/dev/null') // first pass output is not saved as only the .log file is needed
.on('error', reject)
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('end', () => {
// second pass
ffmpeg(input, { niceness: 10 })
@@ -110,7 +122,10 @@ export class MediaRepository implements IMediaRepository {
.addOptions('-pass', '2')
.addOptions('-passlogfile', output)
.output(output)
.on('error', reject)
.on('error', (err, stdout, stderr) => {
this.logger.error(stderr);
reject(err);
})
.on('end', () => fs.unlink(`${output}-0.log`))
.on('end', () => fs.rm(`${output}-0.log.mbtree`, { force: true }))
.on('end', resolve)

2
server/src/microservices/app.service.ts
View File

@@ -60,7 +60,7 @@ export class AppService {
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWebpThumbnail(data),
[JobName.GENERATE_THUMBHASH_THUMBNAIL]: (data) => this.mediaService.handleGenerateThumbhashThumbnail(data),
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),

10
server/test/fixtures.ts
View File

@@ -19,6 +19,7 @@ import {
AssetEntity,
AssetFaceEntity,
AssetType,
AudioCodec,
ExifEntity,
PartnerEntity,
PersonEntity,
@@ -27,9 +28,10 @@ import {
SystemConfig,
TagEntity,
TagType,
TranscodePreset,
TranscodePolicy,
UserEntity,
UserTokenEntity,
VideoCodec,
} from '@app/infra/entities';
const today = new Date();
@@ -685,12 +687,12 @@ export const systemConfigStub = {
crf: 23,
threads: 0,
preset: 'ultrafast',
targetAudioCodec: 'aac',
targetAudioCodec: AudioCodec.AAC,
targetResolution: '720',
targetVideoCodec: 'h264',
targetVideoCodec: VideoCodec.H264,
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
transcode: TranscodePolicy.REQUIRED,
},
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },