KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): transcoding hardware acceleration (#3171)

Browse Source 
* added transcode configs for nvenc,qsv and vaapi

* updated dev docker compose

* added software fallback

* working vaapi

* minor fixes and added tests

* updated api

* compile libvips

* move hwaccel settings to `hwaccel.yml`

* changed default dockerfile, moved `readdir` call

* removed unused import

* minor cleanup

* fix for arm build

* added documentation, minor fixes

* added intel driver

* updated docs

styling

* uppercase codec and api names

* formatting

* added tests

* updated docs

* removed semicolons

* added link to `hwaccel.yml`

* added newlines

* added `hwaccel` section to docker-compose.prod.yml

* ensure mesa drivers are installed

* switch to mimalloc for sharp

* moved build version and sha256 to json

* let libmfx set the render device

* possible fix for vp9 on qsv

* updated tests

* formatting

* review suggestions

* semicolon

* moved `LD_PRELOAD` to start script

* switched to jellyfin's ffmpeg package

* fixed dockerfile

* use cqp instead of icq for qsv vp9

* updated dockerfile

* added sha256sum for other platforms

* fixtures
This commit is contained in:
Mert
2023-08-01 21:56:10 -04:00
committed by GitHub
parent b9cda59172
commit ee49f470b7
44 changed files with 1308 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/domain/media/media.repository.ts
View File

@@ -1,3 +1,5 @@
import { VideoCodec } from '@app/infra/entities';
export const IMediaRepository = 'IMediaRepository';
export interface ResizeOptions {
@@ -55,6 +57,10 @@ export interface VideoCodecSWConfig {
getOptions(stream: VideoStreamInfo): TranscodeOptions;
}
export interface VideoCodecHWConfig extends VideoCodecSWConfig {
getSupportedCodecs(): Array<VideoCodec>;
}
export interface IMediaRepository {
// image
resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;

523
server/src/domain/media/media.service.spec.ts
View File

@@ -1,4 +1,4 @@
import { AssetType, SystemConfigKey, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { AssetType, SystemConfigKey, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import {
assetStub,
newAssetRepositoryMock,
@@ -272,6 +272,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-preset ultrafast',
'-crf 23',
],
@@ -309,6 +310,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-preset ultrafast',
'-crf 23',
],
@@ -331,6 +333,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -357,6 +360,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-preset ultrafast',
'-crf 23',
],
@@ -380,6 +384,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=720:-2',
'-preset ultrafast',
'-crf 23',
@@ -404,6 +409,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -428,6 +434,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -476,6 +483,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -505,6 +513,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-b:v 3104k',
@@ -531,6 +540,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -559,6 +569,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-cpu-used 5',
'-row-mt 1',
@@ -589,6 +600,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-cpu-used 2',
'-row-mt 1',
@@ -618,6 +630,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-row-mt 1',
'-crf 23',
@@ -646,6 +659,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-cpu-used 5',
'-row-mt 1',
@@ -673,6 +687,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-threads 2',
@@ -700,6 +715,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -727,6 +743,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-threads 2',
@@ -757,6 +774,7 @@ describe(MediaService.name, () => {
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
@@ -765,5 +783,508 @@ describe(MediaService.name, () => {
},
);
});
it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => {
mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL },
{ key: SystemConfigKey.FFMPEG_TARGET_RESOLUTION, value: '1080p' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should return false if hwaccel is enabled for an unsupported codec', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should return false if hwaccel option is invalid', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: 'invalid' }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should set two pass options for nvenc when enabled', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
{ key: SystemConfigKey.FFMPEG_TWO_PASS, value: true },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq',
'-qmin 0',
'-g 250',
'-bf 3',
'-b_ref_mode middle',
'-temporal-aq 1',
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf hwupload_cuda,scale_cuda=-2:720',
'-preset p1',
'-b:v 6897k',
'-maxrate 10000k',
'-bufsize 6897k',
'-multipass 2',
],
twoPass: false,
},
);
});
it('should set vbr options for nvenc when max bitrate is enabled', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq',
'-qmin 0',
'-g 250',
'-bf 3',
'-b_ref_mode middle',
'-temporal-aq 1',
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf hwupload_cuda,scale_cuda=-2:720',
'-preset p1',
'-cq:v 23',
'-maxrate 10000k',
'-bufsize 6897k',
],
twoPass: false,
},
);
});
it('should set cq options for nvenc when max bitrate is disabled', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq',
'-qmin 0',
'-g 250',
'-bf 3',
'-b_ref_mode middle',
'-temporal-aq 1',
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf hwupload_cuda,scale_cuda=-2:720',
'-preset p1',
'-cq:v 23',
],
twoPass: false,
},
);
});
it('should omit preset for nvenc if invalid', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq',
'-qmin 0',
'-g 250',
'-bf 3',
'-b_ref_mode middle',
'-temporal-aq 1',
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf hwupload_cuda,scale_cuda=-2:720',
'-cq:v 23',
],
twoPass: false,
},
);
});
it('should ignore two pass for nvenc if max bitrate is disabled', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq',
'-qmin 0',
'-g 250',
'-bf 3',
'-b_ref_mode middle',
'-temporal-aq 1',
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf hwupload_cuda,scale_cuda=-2:720',
'-preset p1',
'-cq:v 23',
],
twoPass: false,
},
);
});
it('should set options for qsv', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [
`-vcodec h264_qsv`,
'-g 256',
'-extbrc 1',
'-refs 5',
'-bf 7',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
'-preset 7',
'-global_quality 23',
'-maxrate 10000k',
'-bufsize 20000k',
],
twoPass: false,
},
);
});
it('should omit preset for qsv if invalid', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [
`-vcodec h264_qsv`,
'-g 256',
'-extbrc 1',
'-refs 5',
'-bf 7',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
'-global_quality 23',
],
twoPass: false,
},
);
});
it('should set low power mode for qsv if target video codec is vp9', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [
`-vcodec vp9_qsv`,
'-g 256',
'-extbrc 1',
'-refs 5',
'-bf 7',
'-low_power 1',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
'-preset 7',
'-q:v 23',
],
twoPass: false,
},
);
});
it('should return false for qsv if no hw devices', async () => {
storageMock.readdir.mockResolvedValue([]);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
it('should set vbr options for vaapi when max bitrate is enabled', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [
`-vcodec h264_vaapi`,
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload,scale_vaapi=-2:720',
'-compression_level 7',
'-b:v 6897k',
'-maxrate 10000k',
'-minrate 3448.5k',
'-rc_mode 3',
],
twoPass: false,
},
);
});
it('should set cq options for vaapi when max bitrate is disabled', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [
`-vcodec h264_vaapi`,
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload,scale_vaapi=-2:720',
'-compression_level 7',
'-qp 23',
'-global_quality 23',
'-rc_mode 1',
],
twoPass: false,
},
);
});
it('should omit preset for vaapi if invalid', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [
`-vcodec h264_vaapi`,
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload,scale_vaapi=-2:720',
'-qp 23',
'-global_quality 23',
'-rc_mode 1',
],
twoPass: false,
},
);
});
it('should prefer gpu for vaapi if available', async () => {
storageMock.readdir.mockResolvedValue(['renderD129', 'card1', 'card0', 'renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'],
outputOptions: [
`-vcodec h264_vaapi`,
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload,scale_vaapi=-2:720',
'-compression_level 7',
'-qp 23',
'-global_quality 23',
'-rc_mode 1',
],
twoPass: false,
},
);
storageMock.readdir.mockResolvedValue(['renderD129', 'renderD128']);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'],
outputOptions: [
`-vcodec h264_vaapi`,
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf format=nv12,hwupload,scale_vaapi=-2:720',
'-compression_level 7',
'-qp 23',
'-global_quality 23',
'-rc_mode 1',
],
twoPass: false,
},
);
});
it('should fallback to sw transcoding if hw transcoding fails', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
mediaMock.transcode.mockRejectedValueOnce(new Error('error'));
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledTimes(2);
expect(mediaMock.transcode).toHaveBeenLastCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-vcodec h264',
'-acodec aac',
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
'-vf scale=-2:720',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
it('should return false for vaapi if no hw devices', async () => {
storageMock.readdir.mockResolvedValue([]);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
});
});

72
server/src/domain/media/media.service.ts
View File

@@ -1,4 +1,4 @@
import { AssetEntity, AssetType, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { AssetEntity, AssetType, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common';
import { join } from 'path';
import { IAssetRepository, WithoutProperty } from '../asset';
@@ -8,8 +8,8 @@ import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
import { JPEG_THUMBNAIL_SIZE, WEBP_THUMBNAIL_SIZE } from './media.constant';
import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository';
import { H264Config, HEVCConfig, VP9Config } from './media.util';
import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository';
import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, VAAPIConfig, VP9Config } from './media.util';
@Injectable()
export class MediaService {
@@ -155,14 +155,26 @@ export class MediaService {
let transcodeOptions;
try {
transcodeOptions = this.getCodecConfig(config).getOptions(mainVideoStream);
transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream));
} catch (err) {
this.logger.error(`An error occurred while configuring transcoding options: ${err}`);
return false;
}
this.logger.log(`Start encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`);
await this.mediaRepository.transcode(input, output, transcodeOptions);
try {
await this.mediaRepository.transcode(input, output, transcodeOptions);
} catch (err) {
this.logger.error(err);
if (config.accel && config.accel !== TranscodeHWAccel.DISABLED) {
this.logger.error(
`Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`,
);
}
config.accel = TranscodeHWAccel.DISABLED;
transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream));
await this.mediaRepository.transcode(input, output, transcodeOptions);
}
this.logger.log(`Encoding success ${asset.id}`);
@@ -195,15 +207,11 @@ export class MediaService {
const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension);
const isTargetAudioCodec = audioStream == null || audioStream.codecName === ffmpegConfig.targetAudioCodec;
if (audioStream != null) {
this.logger.verbose(
`${asset.id}: AudioCodecName ${audioStream.codecName}, AudioStreamCodecType ${audioStream.codecType}, containerExtension ${containerExtension}`,
);
} else {
this.logger.verbose(
`${asset.id}: AudioCodecName None, AudioStreamCodecType None, containerExtension ${containerExtension}`,
);
}
this.logger.verbose(
`${asset.id}: AudioCodecName ${audioStream?.codecName ?? 'None'}, AudioStreamCodecType ${
audioStream?.codecType ?? 'None'
}, containerExtension ${containerExtension}`,
);
const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer;
const scalingEnabled = ffmpegConfig.targetResolution !== 'original';
@@ -228,7 +236,14 @@ export class MediaService {
}
}
private getCodecConfig(config: SystemConfigFFmpegDto) {
async getCodecConfig(config: SystemConfigFFmpegDto) {
if (config.accel === TranscodeHWAccel.DISABLED) {
return this.getSWCodecConfig(config);
}
return this.getHWCodecConfig(config);
}
private getSWCodecConfig(config: SystemConfigFFmpegDto) {
switch (config.targetVideoCodec) {
case VideoCodec.H264:
return new H264Config(config);
@@ -240,4 +255,31 @@ export class MediaService {
throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
}
}
private async getHWCodecConfig(config: SystemConfigFFmpegDto) {
let handler: VideoCodecHWConfig;
let devices: string[];
switch (config.accel) {
case TranscodeHWAccel.NVENC:
handler = new NVENCConfig(config);
break;
case TranscodeHWAccel.QSV:
devices = await this.storageRepository.readdir('/dev/dri');
handler = new QSVConfig(config, devices);
break;
case TranscodeHWAccel.VAAPI:
devices = await this.storageRepository.readdir('/dev/dri');
handler = new VAAPIConfig(config, devices);
break;
default:
throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`);
}
if (!handler.getSupportedCodecs().includes(config.targetVideoCodec)) {
throw new UnsupportedMediaTypeException(
`${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${handler.getSupportedCodecs()}`,
);
}
return handler;
}
}

228
server/src/domain/media/media.util.ts
View File

@@ -1,13 +1,26 @@
import { TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
import { SystemConfigFFmpegDto } from '../system-config/dto';
import { BitrateDistribution, TranscodeOptions, VideoCodecSWConfig, VideoStreamInfo } from './media.repository';
import {
BitrateDistribution,
TranscodeOptions,
VideoCodecHWConfig,
VideoCodecSWConfig,
VideoStreamInfo,
} from './media.repository';
class BaseConfig implements VideoCodecSWConfig {
constructor(protected config: SystemConfigFFmpegDto) {}
getOptions(stream: VideoStreamInfo) {
const options = {
inputOptions: this.getBaseInputOptions(),
outputOptions: this.getBaseOutputOptions(),
outputOptions: this.getBaseOutputOptions().concat([
`-acodec ${this.config.targetAudioCodec}`,
// Makes a second pass moving the moov atom to the
// beginning of the file for improved playback speed.
'-movflags faststart',
'-fps_mode passthrough',
'-v verbose',
]),
twoPass: this.eligibleForTwoPass(),
} as TranscodeOptions;
const filters = this.getFilterOptions(stream);
@@ -26,14 +39,7 @@ class BaseConfig implements VideoCodecSWConfig {
}
getBaseOutputOptions() {
return [
`-vcodec ${this.config.targetVideoCodec}`,
`-acodec ${this.config.targetAudioCodec}`,
// Makes a second pass moving the moov atom to the beginning of
// the file for improved playback speed.
'-movflags faststart',
'-fps_mode passthrough',
];
return [`-vcodec ${this.config.targetVideoCodec}`];
}
getFilterOptions(stream: VideoStreamInfo) {
@@ -77,11 +83,11 @@ class BaseConfig implements VideoCodecSWConfig {
}
eligibleForTwoPass() {
if (!this.config.twoPass) {
if (!this.config.twoPass || this.config.accel !== TranscodeHWAccel.DISABLED) {
return false;
}
return this.isBitrateConstrained() || this.config.targetVideoCodec === 'vp9';
return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
}
getBitrateDistribution() {
@@ -107,7 +113,8 @@ class BaseConfig implements VideoCodecSWConfig {
getScaling(stream: VideoStreamInfo) {
const targetResolution = this.getTargetResolution(stream);
return this.isVideoVertical(stream) ? `${targetResolution}:-2` : `-2:${targetResolution}`;
const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
}
isVideoRotated(stream: VideoStreamInfo) {
@@ -137,6 +144,34 @@ class BaseConfig implements VideoCodecSWConfig {
}
}
export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
protected devices: string[];
constructor(protected config: SystemConfigFFmpegDto, devices: string[] = []) {
super(config);
this.devices = this.validateDevices(devices);
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
}
validateDevices(devices: string[]) {
return devices
.filter((device) => device.startsWith('renderD') || device.startsWith('card'))
.sort((a, b) => {
// order GPU devices first
if (a.startsWith('card') && b.startsWith('renderD')) {
return -1;
}
if (a.startsWith('renderD') && b.startsWith('card')) {
return 1;
}
return -a.localeCompare(b);
});
}
}
export class H264Config extends BaseConfig {
getThreadOptions() {
if (this.config.threads <= 0) {
@@ -189,3 +224,168 @@ export class VP9Config extends BaseConfig {
return ['-row-mt 1', ...super.getThreadOptions()];
}
}
export class NVENCConfig extends BaseHWConfig {
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC];
}
getBaseInputOptions() {
return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
}
getBaseOutputOptions() {
return [
`-vcodec ${this.config.targetVideoCodec}_nvenc`,
// below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
'-tune hq',
'-qmin 0',
'-g 250',
'-bf 3',
'-b_ref_mode middle',
'-temporal-aq 1',
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
];
}
getFilterOptions(stream: VideoStreamInfo) {
const options = ['hwupload_cuda'];
if (this.shouldScale(stream)) {
options.push(`scale_cuda=${this.getScaling(stream)}`);
}
return options;
}
getPresetOptions() {
let presetIndex = this.getPresetIndex();
if (presetIndex < 0) {
return [];
}
presetIndex = 7 - Math.min(6, presetIndex); // map to p1-p7; p7 is the highest quality, so reverse index
return [`-preset p${presetIndex}`];
}
getBitrateOptions() {
const bitrates = this.getBitrateDistribution();
if (bitrates.max > 0 && this.config.twoPass) {
return [
`-b:v ${bitrates.target}${bitrates.unit}`,
`-maxrate ${bitrates.max}${bitrates.unit}`,
`-bufsize ${bitrates.target}${bitrates.unit}`,
'-multipass 2',
];
} else if (bitrates.max > 0) {
return [
`-cq:v ${this.config.crf}`,
`-maxrate ${bitrates.max}${bitrates.unit}`,
`-bufsize ${bitrates.target}${bitrates.unit}`,
];
} else {
return [`-cq:v ${this.config.crf}`];
}
}
getThreadOptions() {
return [];
}
}
export class QSVConfig extends BaseHWConfig {
getBaseInputOptions() {
if (!this.devices.length) {
throw Error('No QSV device found');
}
return ['-init_hw_device qsv=hw', '-filter_hw_device hw'];
}
getBaseOutputOptions() {
// recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
const options = [`-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7'];
// VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
if (this.config.targetVideoCodec === VideoCodec.VP9) {
options.push('-low_power 1');
}
return options;
}
getFilterOptions(stream: VideoStreamInfo) {
const options = ['format=nv12', 'hwupload=extra_hw_frames=64'];
if (this.shouldScale(stream)) {
options.push(`scale_qsv=${this.getScaling(stream)}`);
}
return options;
}
getPresetOptions() {
let presetIndex = this.getPresetIndex();
if (presetIndex < 0) {
return [];
}
presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
return [`-preset ${presetIndex}`];
}
getBitrateOptions() {
const options = [];
if (this.config.targetVideoCodec !== VideoCodec.VP9) {
options.push(`-global_quality ${this.config.crf}`);
} else {
options.push(`-q:v ${this.config.crf}`);
}
const bitrates = this.getBitrateDistribution();
if (bitrates.max > 0) {
options.push(`-maxrate ${bitrates.max}${bitrates.unit}`);
options.push(`-bufsize ${bitrates.max * 2}${bitrates.unit}`);
}
return options;
}
}
export class VAAPIConfig extends BaseHWConfig {
getBaseInputOptions() {
if (this.devices.length === 0) {
throw Error('No VAAPI device found');
}
return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel'];
}
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}_vaapi`];
}
getFilterOptions(stream: VideoStreamInfo) {
const options = ['format=nv12', 'hwupload'];
if (this.shouldScale(stream)) {
options.push(`scale_vaapi=${this.getScaling(stream)}`);
}
return options;
}
getPresetOptions() {
let presetIndex = this.getPresetIndex();
if (presetIndex < 0) {
return [];
}
presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
return [`-compression_level ${presetIndex}`];
}
getBitrateOptions() {
const bitrates = this.getBitrateDistribution();
// VAAPI doesn't allow setting both quality and max bitrate
if (bitrates.max > 0) {
return [
`-b:v ${bitrates.target}${bitrates.unit}`,
`-maxrate ${bitrates.max}${bitrates.unit}`,
`-minrate ${bitrates.min}${bitrates.unit}`,
'-rc_mode 3',
]; // variable bitrate
} else {
return [`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1']; // constant quality
}
}
}

1
server/src/domain/storage/storage.repository.ts
View File

@@ -29,4 +29,5 @@ export interface IStorageRepository {
checkFileExists(filepath: string, mode?: number): Promise<boolean>;
mkdirSync(filepath: string): void;
checkDiskUsage(folder: string): Promise<DiskUsage>;
readdir(folder: string): Promise<string[]>;
}

6
server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts
View File

@@ -1,4 +1,4 @@
import { AudioCodec, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { AudioCodec, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsBoolean, IsEnum, IsInt, IsString, Max, Min } from 'class-validator';
@@ -40,4 +40,8 @@ export class SystemConfigFFmpegDto {
@IsEnum(TranscodePolicy)
@ApiProperty({ enumName: 'TranscodePolicy', enum: TranscodePolicy })
transcode!: TranscodePolicy;
@IsEnum(TranscodeHWAccel)
@ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel })
accel!: TranscodeHWAccel;
}

2
server/src/domain/system-config/system-config.core.ts
View File

@@ -4,6 +4,7 @@ import {
SystemConfigEntity,
SystemConfigKey,
SystemConfigValue,
TranscodeHWAccel,
TranscodePolicy,
VideoCodec,
} from '@app/infra/entities';
@@ -27,6 +28,7 @@ export const defaults = Object.freeze<SystemConfig>({
maxBitrate: '0',
twoPass: false,
transcode: TranscodePolicy.REQUIRED,
accel: TranscodeHWAccel.DISABLED,
},
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },

2
server/src/domain/system-config/system-config.service.spec.ts
View File

@@ -3,6 +3,7 @@ import {
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
TranscodeHWAccel,
TranscodePolicy,
VideoCodec,
} from '@app/infra/entities';
@@ -41,6 +42,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
maxBitrate: '0',
twoPass: false,
transcode: TranscodePolicy.REQUIRED,
accel: TranscodeHWAccel.DISABLED,
},
oauth: {
autoLaunch: true,