feat(all): ffmpeg quality options improvements (#2161)

* feat: change target scaling to resolution in ffmpeg config

* feat(microservices): scale vertical video correctly, only scale if video is larger than target

server/apps/microservices/src/processors/video-transcode.processor.ts

@@ -16,7 +16,7 @@ import { AssetEntity, AssetType, TranscodePreset } from '@app/infra/entities';
 import { Process, Processor } from '@nestjs/bull';
 import { Inject, Logger } from '@nestjs/common';
 import { Job } from 'bull';
-import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
+import ffmpeg, { FfprobeData, FfprobeStream } from 'fluent-ffmpeg';
 import { join } from 'path';
 
 @Processor(QueueName.VIDEO_CONVERSION)
@@ -74,22 +74,22 @@ export class VideoTranscodeProcessor {
 
   async runVideoEncode(asset: AssetEntity, savedEncodedPath: string): Promise<void> {
     const config = await this.systemConfigService.getConfig();
+    const videoStream = await this.getVideoStream(asset);
 
-    const transcode = await this.needsTranscoding(asset, config.ffmpeg);
+    const transcode = await this.needsTranscoding(videoStream, config.ffmpeg);
     if (transcode) {
       //TODO: If video or audio are already the correct format, don't re-encode, copy the stream
-      return this.runFFMPEGPipeLine(asset, savedEncodedPath);
+      return this.runFFMPEGPipeLine(asset, videoStream, savedEncodedPath);
     }
   }
 
-  async needsTranscoding(asset: AssetEntity, ffmpegConfig: SystemConfigFFmpegDto): Promise<boolean> {
+  async needsTranscoding(videoStream: FfprobeStream, ffmpegConfig: SystemConfigFFmpegDto): Promise<boolean> {
     switch (ffmpegConfig.transcode) {
       case TranscodePreset.ALL:
         return true;
 
       case TranscodePreset.REQUIRED:
         {
-          const videoStream = await this.getVideoStream(asset);
           if (videoStream.codec_name !== ffmpegConfig.targetVideoCodec) {
             return true;
           }
@@ -97,12 +97,13 @@ export class VideoTranscodeProcessor {
         break;
 
       case TranscodePreset.OPTIMAL: {
-        const videoStream = await this.getVideoStream(asset);
         if (videoStream.codec_name !== ffmpegConfig.targetVideoCodec) {
           return true;
         }
 
-        const videoHeightThreshold = 1080;
+        const config = await this.systemConfigService.getConfig();
+
+        const videoHeightThreshold = Number.parseInt(config.ffmpeg.targetResolution);
         return !videoStream.height || videoStream.height > videoHeightThreshold;
       }
     }
@@ -125,22 +126,45 @@ export class VideoTranscodeProcessor {
     return longestVideoStream;
   }
 
-  async runFFMPEGPipeLine(asset: AssetEntity, savedEncodedPath: string): Promise<void> {
+  async runFFMPEGPipeLine(asset: AssetEntity, videoStream: FfprobeStream, savedEncodedPath: string): Promise<void> {
     const config = await this.systemConfigService.getConfig();
 
+    const ffmpegOptions = [
+      `-crf ${config.ffmpeg.crf}`,
+      `-preset ${config.ffmpeg.preset}`,
+      `-vcodec ${config.ffmpeg.targetVideoCodec}`,
+      `-acodec ${config.ffmpeg.targetAudioCodec}`,
+      // Makes a second pass moving the moov atom to the beginning of
+      // the file for improved playback speed.
+      `-movflags faststart`,
+    ];
+
+    if (!videoStream.height || !videoStream.width) {
+      this.logger.error('Height or width undefined for video stream');
+      return;
+    }
+
+    const streamHeight = videoStream.height;
+    const streamWidth = videoStream.width;
+
+    const targetResolution = Number.parseInt(config.ffmpeg.targetResolution);
+
+    let scaling = `-2:${targetResolution}`;
+    const shouldScale = Math.min(streamHeight, streamWidth) > targetResolution;
+
+    const videoIsRotated = Math.abs(Number.parseInt(`${videoStream.rotation ?? 0}`)) === 90;
+
+    if (streamHeight > streamWidth || videoIsRotated) {
+      scaling = `${targetResolution}:-2`;
+    }
+
+    if (shouldScale) {
+      ffmpegOptions.push(`-vf scale=${scaling}`);
+    }
+
     return new Promise((resolve, reject) => {
       ffmpeg(asset.originalPath)
-        .outputOptions([
-          `-crf ${config.ffmpeg.crf}`,
-          `-preset ${config.ffmpeg.preset}`,
-          `-vcodec ${config.ffmpeg.targetVideoCodec}`,
-          `-acodec ${config.ffmpeg.targetAudioCodec}`,
-          `-vf scale=${config.ffmpeg.targetScaling}`,
-
-          // Makes a second pass moving the moov atom to the beginning of
-          // the file for improved playback speed.
-          `-movflags faststart`,
-        ])
+        .outputOptions(ffmpegOptions)
         .output(savedEncodedPath)
         .on('start', () => {
           this.logger.log('Start Converting Video');

server/immich-openapi-specs.json

@@ -4644,7 +4644,7 @@
           "targetAudioCodec": {
             "type": "string"
           },
-          "targetScaling": {
+          "targetResolution": {
             "type": "string"
           },
           "transcode": {
@@ -4661,7 +4661,7 @@
           "preset",
           "targetVideoCodec",
           "targetAudioCodec",
-          "targetScaling",
+          "targetResolution",
           "transcode"
         ]
       },

server/libs/domain/src/system-config/dto/system-config-ffmpeg.dto.ts

@@ -15,7 +15,7 @@ export class SystemConfigFFmpegDto {
   targetAudioCodec!: string;
 
   @IsString()
-  targetScaling!: string;
+  targetResolution!: string;
 
   @IsEnum(TranscodePreset)
   transcode!: TranscodePreset;

server/libs/domain/src/system-config/system-config.core.ts

@@ -13,7 +13,7 @@ const defaults: SystemConfig = Object.freeze({
     preset: 'ultrafast',
     targetVideoCodec: 'h264',
     targetAudioCodec: 'aac',
-    targetScaling: '1280:-2',
+    targetResolution: '720',
     transcode: TranscodePreset.REQUIRED,
   },
   oauth: {

server/libs/domain/src/system-config/system-config.service.spec.ts

@@ -16,7 +16,7 @@ const updatedConfig = Object.freeze({
     crf: 'a new value',
     preset: 'ultrafast',
     targetAudioCodec: 'aac',
-    targetScaling: '1280:-2',
+    targetResolution: '720',
     targetVideoCodec: 'h264',
     transcode: TranscodePreset.REQUIRED,
   },

server/libs/domain/test/fixtures.ts

@@ -401,7 +401,7 @@ export const systemConfigStub = {
       crf: '23',
       preset: 'ultrafast',
       targetAudioCodec: 'aac',
-      targetScaling: '1280:-2',
+      targetResolution: '720',
       targetVideoCodec: 'h264',
       transcode: TranscodePreset.REQUIRED,
     },

server/libs/infra/src/entities/system-config.entity.ts

@@ -17,7 +17,7 @@ export enum SystemConfigKey {
   FFMPEG_PRESET = 'ffmpeg.preset',
   FFMPEG_TARGET_VIDEO_CODEC = 'ffmpeg.targetVideoCodec',
   FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec',
-  FFMPEG_TARGET_SCALING = 'ffmpeg.targetScaling',
+  FFMPEG_TARGET_RESOLUTION = 'ffmpeg.targetResolution',
   FFMPEG_TRANSCODE = 'ffmpeg.transcode',
   OAUTH_ENABLED = 'oauth.enabled',
   OAUTH_ISSUER_URL = 'oauth.issuerUrl',
@@ -45,7 +45,7 @@ export interface SystemConfig {
     preset: string;
     targetVideoCodec: string;
     targetAudioCodec: string;
-    targetScaling: string;
+    targetResolution: string;
     transcode: TranscodePreset;
   };
   oauth: {