feat(server): transcode bitrate and thread settings (#2488)

* support for two-pass transcoding

* added max bitrate and thread to transcode api

* admin page setting desc+bitrate and thread options

* Update web/src/lib/components/admin-page/settings/setting-input-field.svelte

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Update web/src/lib/components/admin-page/settings/setting-input-field.svelte

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* two-pass slider, `crf` and `threads` as numbers

* updated and added transcode tests

* refactored `getFfmpegOptions`

* default `threads`, `maxBitrate` now 0, more tests

* vp9 constant quality mode

* fixed nullable `crf` and `threads`

* fixed two-pass slider, added apiproperty

* optional `desc` for `SettingSelect`

* disable two-pass if settings are incompatible

* fixed test

* transcode interface

---------

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
This commit is contained in:
Mert
2023-05-22 14:07:43 -04:00
committed by GitHub
parent f1384fea58
commit e9722710ac
19 changed files with 498 additions and 46 deletions

View File

@@ -38,6 +38,11 @@ export interface CropOptions {
height: number;
}
export interface TranscodeOptions {
outputOptions: string[];
twoPass: boolean;
}
export interface IMediaRepository {
// image
extractThumbnailFromExif(input: string, output: string): Promise<void>;
@@ -47,5 +52,5 @@ export interface IMediaRepository {
// video
extractVideoThumbnail(input: string, output: string, size: number): Promise<void>;
probe(input: string): Promise<VideoInfo>;
transcode(input: string, output: string, options: any): Promise<void>;
transcode(input: string, output: string, options: TranscodeOptions): Promise<void>;
}

View File

@@ -253,7 +253,10 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart'],
{
outputOptions: ['-vcodec h264', '-acodec aac', '-movflags faststart', '-preset ultrafast', '-crf 23'],
twoPass: false,
},
);
});
@@ -276,7 +279,10 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart'],
{
outputOptions: ['-vcodec h264', '-acodec aac', '-movflags faststart', '-preset ultrafast', '-crf 23'],
twoPass: false,
},
);
});
@@ -287,7 +293,17 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'],
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
@@ -298,7 +314,17 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=720:-2'],
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=720:-2',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
@@ -309,7 +335,17 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'],
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
@@ -320,7 +356,17 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'],
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
@@ -330,5 +376,152 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should set max bitrate if above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '4500k' }]);
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
'-maxrate 4500k',
],
twoPass: false,
},
);
});
it('should transcode in two passes for h264/h265 when enabled and max bitrate is above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '4500k' },
{ key: SystemConfigKey.FFMPEG_TWO_PASS, value: true },
]);
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-b:v 3104k',
'-minrate 1552k',
'-maxrate 4500k',
],
twoPass: true,
},
);
});
it('should fallback to one pass for h264/h265 if two-pass is enabled but no max bitrate is set', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TWO_PASS, value: true }]);
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
it('should configure preset for vp9', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: 'vp9' },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 },
]);
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
outputOptions: [
'-vcodec vp9',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-cpu-used 5',
'-row-mt 1',
'-threads 2',
'-crf 23',
'-b:v 0',
],
twoPass: false,
},
);
});
it('should configure threads if above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: 'vp9' },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 },
]);
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
outputOptions: [
'-vcodec vp9',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-cpu-used 5',
'-row-mt 1',
'-threads 2',
'-crf 23',
'-b:v 0',
],
twoPass: false,
},
);
});
it('should disable thread pooling for x264/x265 if thread limit is above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 }]);
await sut.handleVideoConversion({ asset: assetEntityStub.video });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-vf scale=-2:720',
'-preset ultrafast',
'-threads 2',
'-x264-params "pools=none"',
'-x264-params "frame-threads=2"',
'-crf 23',
],
twoPass: false,
},
);
});
});
});

View File

@@ -165,10 +165,11 @@ export class MediaService {
return;
}
const options = this.getFfmpegOptions(mainVideoStream, config);
const outputOptions = this.getFfmpegOptions(mainVideoStream, config);
const twoPass = this.eligibleForTwoPass(config);
this.logger.log(`Start encoding video ${asset.id} ${options}`);
await this.mediaRepository.transcode(input, output, options);
this.logger.log(`Start encoding video ${asset.id} ${outputOptions}`);
await this.mediaRepository.transcode(input, output, { outputOptions, twoPass });
this.logger.log(`Encoding success ${asset.id}`);
@@ -231,8 +232,6 @@ export class MediaService {
private getFfmpegOptions(stream: VideoStreamInfo, ffmpeg: SystemConfigFFmpegDto) {
const options = [
`-crf ${ffmpeg.crf}`,
`-preset ${ffmpeg.preset}`,
`-vcodec ${ffmpeg.targetVideoCodec}`,
`-acodec ${ffmpeg.targetAudioCodec}`,
// Makes a second pass moving the moov atom to the beginning of
@@ -240,17 +239,81 @@ export class MediaService {
`-movflags faststart`,
];
// video dimensions
const videoIsRotated = Math.abs(stream.rotation) === 90;
const targetResolution = Number.parseInt(ffmpeg.targetResolution);
const isVideoVertical = stream.height > stream.width || videoIsRotated;
const scaling = isVideoVertical ? `${targetResolution}:-2` : `-2:${targetResolution}`;
const shouldScale = Math.min(stream.height, stream.width) > targetResolution;
// video codec
const isVP9 = ffmpeg.targetVideoCodec === 'vp9';
const isH264 = ffmpeg.targetVideoCodec === 'h264';
const isH265 = ffmpeg.targetVideoCodec === 'hevc';
// transcode efficiency
const limitThreads = ffmpeg.threads > 0;
const maxBitrateValue = Number.parseInt(ffmpeg.maxBitrate) || 0;
const constrainMaximumBitrate = maxBitrateValue > 0;
const bitrateUnit = ffmpeg.maxBitrate.trim().substring(maxBitrateValue.toString().length); // use inputted unit if provided
if (shouldScale) {
options.push(`-vf scale=${scaling}`);
}
if (isH264 || isH265) {
options.push(`-preset ${ffmpeg.preset}`);
}
if (isVP9) {
// vp9 doesn't have presets, but does have a similar setting -cpu-used, from 0-5, 0 being the slowest
const presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
const speed = Math.min(presets.indexOf(ffmpeg.preset), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
if (speed >= 0) {
options.push(`-cpu-used ${speed}`);
}
options.push('-row-mt 1'); // better multithreading
}
if (limitThreads) {
options.push(`-threads ${ffmpeg.threads}`);
// x264 and x265 handle threads differently than one might expect
// https://x265.readthedocs.io/en/latest/cli.html#cmdoption-pools
if (isH264 || isH265) {
options.push(`-${isH265 ? 'x265' : 'x264'}-params "pools=none"`);
options.push(`-${isH265 ? 'x265' : 'x264'}-params "frame-threads=${ffmpeg.threads}"`);
}
}
// two-pass mode for x264/x265 uses bitrate ranges, so it requires a max bitrate from which to derive a target and min bitrate
if (constrainMaximumBitrate && ffmpeg.twoPass) {
const targetBitrateValue = Math.ceil(maxBitrateValue / 1.45); // recommended by https://developers.google.com/media/vp9/settings/vod
const minBitrateValue = targetBitrateValue / 2;
options.push(`-b:v ${targetBitrateValue}${bitrateUnit}`);
options.push(`-minrate ${minBitrateValue}${bitrateUnit}`);
options.push(`-maxrate ${maxBitrateValue}${bitrateUnit}`);
} else if (constrainMaximumBitrate || isVP9) {
// for vp9, these flags work for both one-pass and two-pass
options.push(`-crf ${ffmpeg.crf}`);
options.push(`${isVP9 ? '-b:v' : '-maxrate'} ${maxBitrateValue}${bitrateUnit}`);
} else {
options.push(`-crf ${ffmpeg.crf}`);
}
return options;
}
private eligibleForTwoPass(ffmpeg: SystemConfigFFmpegDto) {
if (!ffmpeg.twoPass) {
return false;
}
const isVP9 = ffmpeg.targetVideoCodec === 'vp9';
const maxBitrateValue = Number.parseInt(ffmpeg.maxBitrate) || 0;
const constrainMaximumBitrate = maxBitrateValue > 0;
return constrainMaximumBitrate || isVP9;
}
}

View File

@@ -1,9 +1,21 @@
import { IsEnum, IsString } from 'class-validator';
import { IsEnum, IsString, IsInt, IsBoolean, Min, Max } from 'class-validator';
import { TranscodePreset } from '@app/infra/entities';
import { Type } from 'class-transformer';
import { ApiProperty } from '@nestjs/swagger';
export class SystemConfigFFmpegDto {
@IsString()
crf!: string;
@IsInt()
@Min(0)
@Max(51)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
crf!: number;
@IsInt()
@Min(0)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
threads!: number;
@IsString()
preset!: string;
@@ -17,6 +29,12 @@ export class SystemConfigFFmpegDto {
@IsString()
targetResolution!: string;
@IsString()
maxBitrate!: string;
@IsBoolean()
twoPass!: boolean;
@IsEnum(TranscodePreset)
transcode!: TranscodePreset;
}

View File

@@ -9,11 +9,14 @@ export type SystemConfigValidator = (config: SystemConfig) => void | Promise<voi
const defaults: SystemConfig = Object.freeze({
ffmpeg: {
crf: '23',
crf: 23,
threads: 0,
preset: 'ultrafast',
targetVideoCodec: 'h264',
targetAudioCodec: 'aac',
targetResolution: '720',
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
oauth: {

View File

@@ -7,17 +7,20 @@ import { ISystemConfigRepository } from './system-config.repository';
import { SystemConfigService } from './system-config.service';
const updates: SystemConfigEntity[] = [
{ key: SystemConfigKey.FFMPEG_CRF, value: 'a new value' },
{ key: SystemConfigKey.FFMPEG_CRF, value: 30 },
{ key: SystemConfigKey.OAUTH_AUTO_LAUNCH, value: true },
];
const updatedConfig = Object.freeze({
ffmpeg: {
crf: 'a new value',
crf: 30,
threads: 0,
preset: 'ultrafast',
targetAudioCodec: 'aac',
targetResolution: '720',
targetVideoCodec: 'h264',
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
oauth: {
@@ -85,7 +88,7 @@ describe(SystemConfigService.name, () => {
it('should merge the overrides', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_CRF, value: 'a new value' },
{ key: SystemConfigKey.FFMPEG_CRF, value: 30 },
{ key: SystemConfigKey.OAUTH_AUTO_LAUNCH, value: true },
]);

View File

@@ -479,11 +479,14 @@ export const keyStub = {
export const systemConfigStub = {
defaults: Object.freeze({
ffmpeg: {
crf: '23',
crf: 23,
threads: 0,
preset: 'ultrafast',
targetAudioCodec: 'aac',
targetResolution: '720',
targetVideoCodec: 'h264',
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
oauth: {

View File

@@ -1,7 +1,7 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
@Entity('system_config')
export class SystemConfigEntity<T = string | boolean> {
export class SystemConfigEntity<T = string | boolean | number> {
@PrimaryColumn()
key!: SystemConfigKey;
@@ -14,10 +14,13 @@ export type SystemConfigValue = any;
// dot notation matches path in `SystemConfig`
export enum SystemConfigKey {
FFMPEG_CRF = 'ffmpeg.crf',
FFMPEG_THREADS = 'ffmpeg.threads',
FFMPEG_PRESET = 'ffmpeg.preset',
FFMPEG_TARGET_VIDEO_CODEC = 'ffmpeg.targetVideoCodec',
FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec',
FFMPEG_TARGET_RESOLUTION = 'ffmpeg.targetResolution',
FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate',
FFMPEG_TWO_PASS = 'ffmpeg.twoPass',
FFMPEG_TRANSCODE = 'ffmpeg.transcode',
OAUTH_ENABLED = 'oauth.enabled',
OAUTH_ISSUER_URL = 'oauth.issuerUrl',
@@ -42,11 +45,14 @@ export enum TranscodePreset {
export interface SystemConfig {
ffmpeg: {
crf: string;
crf: number;
threads: number;
preset: string;
targetVideoCodec: string;
targetAudioCodec: string;
targetResolution: string;
maxBitrate: string;
twoPass: boolean;
transcode: TranscodePreset;
};
oauth: {

View File

@@ -1,8 +1,9 @@
import { CropOptions, IMediaRepository, ResizeOptions, VideoInfo } from '@app/domain';
import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain';
import { exiftool } from 'exiftool-vendored';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import sharp from 'sharp';
import { promisify } from 'util';
import fs from 'fs/promises';
const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
@@ -85,14 +86,40 @@ export class MediaRepository implements IMediaRepository {
};
}
transcode(input: string, output: string, options: string[]): Promise<void> {
transcode(input: string, output: string, options: TranscodeOptions): Promise<void> {
if (!options.twoPass) {
return new Promise((resolve, reject) => {
ffmpeg(input, { niceness: 10 })
.outputOptions(options.outputOptions)
.output(output)
.on('error', reject)
.on('end', resolve)
.run();
});
}
// two-pass allows for precise control of bitrate at the cost of running twice
// recommended for vp9 for better quality and compression
return new Promise((resolve, reject) => {
ffmpeg(input, { niceness: 10 })
//
.outputOptions(options)
.output(output)
.outputOptions(options.outputOptions)
.addOptions('-pass', '1')
.addOptions('-passlogfile', output)
.addOptions('-f null')
.output('/dev/null') // first pass output is not saved as only the .log file is needed
.on('error', reject)
.on('end', resolve)
.on('end', () => {
// second pass
ffmpeg(input, { niceness: 10 })
.outputOptions(options.outputOptions)
.addOptions('-pass', '2')
.addOptions('-passlogfile', output)
.output(output)
.on('error', reject)
.on('end', () => fs.unlink(`${output}-0.log`))
.on('end', resolve)
.run();
})
.run();
});
}

View File

@@ -9,7 +9,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
private repository: Repository<SystemConfigEntity>,
) {}
load(): Promise<SystemConfigEntity<string | boolean>[]> {
load(): Promise<SystemConfigEntity<string | boolean | number>[]> {
return this.repository.find();
}