feat(server): transcoding hardware acceleration (#3171)

* added transcode configs for nvenc,qsv and vaapi * updated dev docker compose * added software fallback * working vaapi * minor fixes and added tests * updated api * compile libvips * move hwaccel settings to `hwaccel.yml` * changed default dockerfile, moved `readdir` call * removed unused import * minor cleanup * fix for arm build * added documentation, minor fixes * added intel driver * updated docs styling * uppercase codec and api names * formatting * added tests * updated docs * removed semicolons * added link to `hwaccel.yml` * added newlines * added `hwaccel` section to docker-compose.prod.yml * ensure mesa drivers are installed * switch to mimalloc for sharp * moved build version and sha256 to json * let libmfx set the render device * possible fix for vp9 on qsv * updated tests * formatting * review suggestions * semicolon * moved `LD_PRELOAD` to start script * switched to jellyfin's ffmpeg package * fixed dockerfile * use cqp instead of icq for qsv vp9 * updated dockerfile * added sha256sum for other platforms * fixtures
2025-10-29 17:40:28 +00:00 · 2023-08-01 21:56:10 -04:00
parent b9cda59172
commit ee49f470b7
44 changed files with 1308 additions and 68 deletions
--- a/server/src/domain/media/media.util.ts
+++ b/server/src/domain/media/media.util.ts
@@ -1,13 +1,26 @@
+import { TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
 import { SystemConfigFFmpegDto } from '../system-config/dto';
-import { BitrateDistribution, TranscodeOptions, VideoCodecSWConfig, VideoStreamInfo } from './media.repository';
-
+import {
+  BitrateDistribution,
+  TranscodeOptions,
+  VideoCodecHWConfig,
+  VideoCodecSWConfig,
+  VideoStreamInfo,
+} from './media.repository';
 class BaseConfig implements VideoCodecSWConfig {
  constructor(protected config: SystemConfigFFmpegDto) {}

  getOptions(stream: VideoStreamInfo) {
    const options = {
      inputOptions: this.getBaseInputOptions(),
-      outputOptions: this.getBaseOutputOptions(),
+      outputOptions: this.getBaseOutputOptions().concat([
+        `-acodec ${this.config.targetAudioCodec}`,
+        // Makes a second pass moving the moov atom to the
+        // beginning of the file for improved playback speed.
+        '-movflags faststart',
+        '-fps_mode passthrough',
+        '-v verbose',
+      ]),
      twoPass: this.eligibleForTwoPass(),
    } as TranscodeOptions;
    const filters = this.getFilterOptions(stream);
@@ -26,14 +39,7 @@ class BaseConfig implements VideoCodecSWConfig {
  }

  getBaseOutputOptions() {
-    return [
-      `-vcodec ${this.config.targetVideoCodec}`,
-      `-acodec ${this.config.targetAudioCodec}`,
-      // Makes a second pass moving the moov atom to the beginning of
-      // the file for improved playback speed.
-      '-movflags faststart',
-      '-fps_mode passthrough',
-    ];
+    return [`-vcodec ${this.config.targetVideoCodec}`];
  }

  getFilterOptions(stream: VideoStreamInfo) {
@@ -77,11 +83,11 @@ class BaseConfig implements VideoCodecSWConfig {
  }

  eligibleForTwoPass() {
-    if (!this.config.twoPass) {
+    if (!this.config.twoPass || this.config.accel !== TranscodeHWAccel.DISABLED) {
      return false;
    }

-    return this.isBitrateConstrained() || this.config.targetVideoCodec === 'vp9';
+    return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
  }

  getBitrateDistribution() {
@@ -107,7 +113,8 @@ class BaseConfig implements VideoCodecSWConfig {

  getScaling(stream: VideoStreamInfo) {
    const targetResolution = this.getTargetResolution(stream);
-    return this.isVideoVertical(stream) ? `${targetResolution}:-2` : `-2:${targetResolution}`;
+    const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
+    return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
  }

  isVideoRotated(stream: VideoStreamInfo) {
@@ -137,6 +144,34 @@ class BaseConfig implements VideoCodecSWConfig {
  }
 }

+export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
+  protected devices: string[];
+
+  constructor(protected config: SystemConfigFFmpegDto, devices: string[] = []) {
+    super(config);
+    this.devices = this.validateDevices(devices);
+  }
+
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
+  }
+
+  validateDevices(devices: string[]) {
+    return devices
+      .filter((device) => device.startsWith('renderD') || device.startsWith('card'))
+      .sort((a, b) => {
+        // order GPU devices first
+        if (a.startsWith('card') && b.startsWith('renderD')) {
+          return -1;
+        }
+        if (a.startsWith('renderD') && b.startsWith('card')) {
+          return 1;
+        }
+        return -a.localeCompare(b);
+      });
+  }
+}
+
 export class H264Config extends BaseConfig {
  getThreadOptions() {
    if (this.config.threads <= 0) {
@@ -189,3 +224,168 @@ export class VP9Config extends BaseConfig {
    return ['-row-mt 1', ...super.getThreadOptions()];
  }
 }
+
+export class NVENCConfig extends BaseHWConfig {
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC];
+  }
+
+  getBaseInputOptions() {
+    return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
+  }
+
+  getBaseOutputOptions() {
+    return [
+      `-vcodec ${this.config.targetVideoCodec}_nvenc`,
+      // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
+      '-tune hq',
+      '-qmin 0',
+      '-g 250',
+      '-bf 3',
+      '-b_ref_mode middle',
+      '-temporal-aq 1',
+      '-rc-lookahead 20',
+      '-i_qfactor 0.75',
+      '-b_qfactor 1.1',
+    ];
+  }
+
+  getFilterOptions(stream: VideoStreamInfo) {
+    const options = ['hwupload_cuda'];
+    if (this.shouldScale(stream)) {
+      options.push(`scale_cuda=${this.getScaling(stream)}`);
+    }
+
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = 7 - Math.min(6, presetIndex); // map to p1-p7; p7 is the highest quality, so reverse index
+    return [`-preset p${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0 && this.config.twoPass) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.target}${bitrates.unit}`,
+        '-multipass 2',
+      ];
+    } else if (bitrates.max > 0) {
+      return [
+        `-cq:v ${this.config.crf}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.target}${bitrates.unit}`,
+      ];
+    } else {
+      return [`-cq:v ${this.config.crf}`];
+    }
+  }
+
+  getThreadOptions() {
+    return [];
+  }
+}
+
+export class QSVConfig extends BaseHWConfig {
+  getBaseInputOptions() {
+    if (!this.devices.length) {
+      throw Error('No QSV device found');
+    }
+    return ['-init_hw_device qsv=hw', '-filter_hw_device hw'];
+  }
+
+  getBaseOutputOptions() {
+    // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
+    const options = [`-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7'];
+    // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
+    if (this.config.targetVideoCodec === VideoCodec.VP9) {
+      options.push('-low_power 1');
+    }
+    return options;
+  }
+
+  getFilterOptions(stream: VideoStreamInfo) {
+    const options = ['format=nv12', 'hwupload=extra_hw_frames=64'];
+    if (this.shouldScale(stream)) {
+      options.push(`scale_qsv=${this.getScaling(stream)}`);
+    }
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
+    return [`-preset ${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const options = [];
+    if (this.config.targetVideoCodec !== VideoCodec.VP9) {
+      options.push(`-global_quality ${this.config.crf}`);
+    } else {
+      options.push(`-q:v ${this.config.crf}`);
+    }
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0) {
+      options.push(`-maxrate ${bitrates.max}${bitrates.unit}`);
+      options.push(`-bufsize ${bitrates.max * 2}${bitrates.unit}`);
+    }
+    return options;
+  }
+}
+
+export class VAAPIConfig extends BaseHWConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw Error('No VAAPI device found');
+    }
+    return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel'];
+  }
+
+  getBaseOutputOptions() {
+    return [`-vcodec ${this.config.targetVideoCodec}_vaapi`];
+  }
+
+  getFilterOptions(stream: VideoStreamInfo) {
+    const options = ['format=nv12', 'hwupload'];
+    if (this.shouldScale(stream)) {
+      options.push(`scale_vaapi=${this.getScaling(stream)}`);
+    }
+
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
+    return [`-compression_level ${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    // VAAPI doesn't allow setting both quality and max bitrate
+    if (bitrates.max > 0) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-minrate ${bitrates.min}${bitrates.unit}`,
+        '-rc_mode 3',
+      ]; // variable bitrate
+    } else {
+      return [`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1']; // constant quality
+    }
+  }
+}