KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): extract motion photo android single frame (#3903)

Browse Source
This commit is contained in:
Mert
2023-08-29 05:01:42 -04:00
committed by GitHub
parent d0a06739d8
commit e510e733cd
7 changed files with 163 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
server/src/domain/media/media.util.ts
View File

@@ -1,6 +1,7 @@
import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
import { SystemConfigFFmpegDto } from '../system-config/dto';
import {
AudioStreamInfo,
BitrateDistribution,
TranscodeOptions,
VideoCodecHWConfig,
@@ -10,13 +11,13 @@ import {
class BaseConfig implements VideoCodecSWConfig {
constructor(protected config: SystemConfigFFmpegDto) {}
getOptions(stream: VideoStreamInfo) {
getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
const options = {
inputOptions: this.getBaseInputOptions(),
outputOptions: this.getBaseOutputOptions().concat('-v verbose'),
outputOptions: this.getBaseOutputOptions(videoStream, audioStream).concat('-v verbose'),
twoPass: this.eligibleForTwoPass(),
} as TranscodeOptions;
const filters = this.getFilterOptions(stream);
const filters = this.getFilterOptions(videoStream);
if (filters.length > 0) {
options.outputOptions.push(`-vf ${filters.join(',')}`);
}
@@ -31,9 +32,10 @@ class BaseConfig implements VideoCodecSWConfig {
return [];
}
getBaseOutputOptions() {
getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
return [
`-acodec ${this.config.targetAudioCodec}`,
`-c:v:${videoStream.index} ${this.getVideoCodec()}`,
`-c:a:${audioStream.index} ${this.getAudioCodec()}`,
// Makes a second pass moving the moov atom to the
// beginning of the file for improved playback speed.
'-movflags faststart',
@@ -41,13 +43,13 @@ class BaseConfig implements VideoCodecSWConfig {
];
}
getFilterOptions(stream: VideoStreamInfo) {
getFilterOptions(videoStream: VideoStreamInfo) {
const options = [];
if (this.shouldScale(stream)) {
options.push(`scale=${this.getScaling(stream)}`);
if (this.shouldScale(videoStream)) {
options.push(`scale=${this.getScaling(videoStream)}`);
}
if (this.shouldToneMap(stream)) {
if (this.shouldToneMap(videoStream)) {
options.push(...this.getToneMapping());
}
options.push('format=yuv420p');
@@ -103,34 +105,34 @@ class BaseConfig implements VideoCodecSWConfig {
return { max, target, min, unit } as BitrateDistribution;
}
getTargetResolution(stream: VideoStreamInfo) {
getTargetResolution(videoStream: VideoStreamInfo) {
if (this.config.targetResolution === 'original') {
return Math.min(stream.height, stream.width);
return Math.min(videoStream.height, videoStream.width);
}
return Number.parseInt(this.config.targetResolution);
}
shouldScale(stream: VideoStreamInfo) {
return Math.min(stream.height, stream.width) > this.getTargetResolution(stream);
shouldScale(videoStream: VideoStreamInfo) {
return Math.min(videoStream.height, videoStream.width) > this.getTargetResolution(videoStream);
}
shouldToneMap(stream: VideoStreamInfo) {
return stream.isHDR && this.config.tonemap !== ToneMapping.DISABLED;
shouldToneMap(videoStream: VideoStreamInfo) {
return videoStream.isHDR && this.config.tonemap !== ToneMapping.DISABLED;
}
getScaling(stream: VideoStreamInfo) {
const targetResolution = this.getTargetResolution(stream);
getScaling(videoStream: VideoStreamInfo) {
const targetResolution = this.getTargetResolution(videoStream);
const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
}
isVideoRotated(stream: VideoStreamInfo) {
return Math.abs(stream.rotation) === 90;
isVideoRotated(videoStream: VideoStreamInfo) {
return Math.abs(videoStream.rotation) === 90;
}
isVideoVertical(stream: VideoStreamInfo) {
return stream.height > stream.width || this.isVideoRotated(stream);
isVideoVertical(videoStream: VideoStreamInfo) {
return videoStream.height > videoStream.width || this.isVideoRotated(videoStream);
}
isBitrateConstrained() {
@@ -171,6 +173,14 @@ class BaseConfig implements VideoCodecSWConfig {
`zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
];
}
getAudioCodec(): string {
return this.config.targetAudioCodec;
}
getVideoCodec(): string {
return this.config.targetVideoCodec;
}
}
export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
@@ -202,6 +212,10 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
return -a.localeCompare(b);
});
}
getVideoCodec(): string {
return `${this.config.targetVideoCodec}_${this.config.accel}`;
}
}
export class ThumbnailConfig extends BaseConfig {
@@ -217,9 +231,9 @@ export class ThumbnailConfig extends BaseConfig {
return [];
}
getScaling(stream: VideoStreamInfo) {
let options = super.getScaling(stream);
if (!this.shouldToneMap(stream)) {
getScaling(videoStream: VideoStreamInfo) {
let options = super.getScaling(videoStream);
if (!this.shouldToneMap(videoStream)) {
options += ':out_color_matrix=bt601:out_range=pc';
}
return options;
@@ -236,10 +250,6 @@ export class ThumbnailConfig extends BaseConfig {
}
export class H264Config extends BaseConfig {
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
}
getThreadOptions() {
if (this.config.threads <= 0) {
return [];
@@ -253,10 +263,6 @@ export class H264Config extends BaseConfig {
}
export class HEVCConfig extends BaseConfig {
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
}
getThreadOptions() {
if (this.config.threads <= 0) {
return [];
@@ -270,10 +276,6 @@ export class HEVCConfig extends BaseConfig {
}
export class VP9Config extends BaseConfig {
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
}
getPresetOptions() {
const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
if (speed >= 0) {
@@ -309,9 +311,8 @@ export class NVENCConfig extends BaseHWConfig {
return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
}
getBaseOutputOptions() {
getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
return [
`-vcodec ${this.config.targetVideoCodec}_nvenc`,
// below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
'-tune hq',
'-qmin 0',
@@ -322,15 +323,15 @@ export class NVENCConfig extends BaseHWConfig {
'-rc-lookahead 20',
'-i_qfactor 0.75',
'-b_qfactor 1.1',
...super.getBaseOutputOptions(),
...super.getBaseOutputOptions(videoStream, audioStream),
];
}
getFilterOptions(stream: VideoStreamInfo) {
const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
getFilterOptions(videoStream: VideoStreamInfo) {
const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
options.push('format=nv12', 'hwupload_cuda');
if (this.shouldScale(stream)) {
options.push(`scale_cuda=${this.getScaling(stream)}`);
if (this.shouldScale(videoStream)) {
options.push(`scale_cuda=${this.getScaling(videoStream)}`);
}
return options;
@@ -378,15 +379,14 @@ export class QSVConfig extends BaseHWConfig {
return ['-init_hw_device qsv=hw', '-filter_hw_device hw'];
}
getBaseOutputOptions() {
getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
// recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
const options = [
`-vcodec ${this.config.targetVideoCodec}_qsv`,
'-g 256',
'-extbrc 1',
'-refs 5',
'-bf 7',
...super.getBaseOutputOptions(),
...super.getBaseOutputOptions(videoStream, audioStream),
];
// VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
if (this.config.targetVideoCodec === VideoCodec.VP9) {
@@ -395,11 +395,11 @@ export class QSVConfig extends BaseHWConfig {
return options;
}
getFilterOptions(stream: VideoStreamInfo) {
const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
getFilterOptions(videoStream: VideoStreamInfo) {
const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
options.push('format=nv12', 'hwupload=extra_hw_frames=64');
if (this.shouldScale(stream)) {
options.push(`scale_qsv=${this.getScaling(stream)}`);
if (this.shouldScale(videoStream)) {
options.push(`scale_qsv=${this.getScaling(videoStream)}`);
}
return options;
}
@@ -437,15 +437,11 @@ export class VAAPIConfig extends BaseHWConfig {
return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel'];
}
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}_vaapi`, ...super.getBaseOutputOptions()];
}
getFilterOptions(stream: VideoStreamInfo) {
const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
getFilterOptions(videoStream: VideoStreamInfo) {
const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
options.push('format=nv12', 'hwupload');
if (this.shouldScale(stream)) {
options.push(`scale_vaapi=${this.getScaling(stream)}`);
if (this.shouldScale(videoStream)) {
options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
}
return options;