KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-12-08 20:29:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): wide gamut thumbnails (#3658)

Browse Source
This commit is contained in:
Mert
2023-09-03 02:21:51 -04:00
committed by GitHub
parent 4bd77d5899
commit 2069293cc1
27 changed files with 405 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
server/src/domain/facial-recognition/facial-recognition.service.spec.ts
View File

@@ -1,3 +1,4 @@
import { Colorspace } from '@app/infra/entities';
import {
assetStub,
faceStub,
@@ -115,7 +116,6 @@ describe(FacialRecognitionService.name, () => {
personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock();
storageMock = newStorageRepositoryMock();
configMock = newSystemConfigRepositoryMock();
mediaMock.crop.mockResolvedValue(croppedFace);
@@ -292,6 +292,8 @@ describe(FacialRecognitionService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
expect(personMock.update).toHaveBeenCalledWith({
id: 'person-1',
@@ -313,6 +315,8 @@ describe(FacialRecognitionService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
});
@@ -330,6 +334,8 @@ describe(FacialRecognitionService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
});
});

9
server/src/domain/facial-recognition/facial-recognition.services.ts
View File

@@ -162,8 +162,15 @@ export class FacialRecognitionService {
height: newHalfSize * 2,
};
const { thumbnail } = await this.configCore.getConfig();
const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions);
await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' });
const thumbnailOptions = {
format: 'jpeg',
size: FACE_THUMBNAIL_SIZE,
colorspace: thumbnail.colorspace,
quality: thumbnail.quality,
} as const;
await this.mediaRepository.resize(croppedOutput, output, thumbnailOptions);
await this.personRepository.update({ id: personId, thumbnailPath: output });
return true;

5
server/src/domain/media/media.repository.ts
View File

@@ -1,10 +1,13 @@
import { VideoCodec } from '@app/infra/entities';
import { Writable } from 'stream';
export const IMediaRepository = 'IMediaRepository';
export interface ResizeOptions {
size: number;
format: 'webp' | 'jpeg';
colorspace: string;
quality: number;
}
export interface VideoStreamInfo {
@@ -73,5 +76,5 @@ export interface IMediaRepository {
// video
probe(input: string): Promise<VideoInfo>;
transcode(input: string, output: string, options: TranscodeOptions): Promise<void>;
transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void>;
}

26
server/src/domain/media/media.service.spec.ts
View File

@@ -1,5 +1,6 @@
import {
AssetType,
Colorspace,
SystemConfigKey,
ToneMapping,
TranscodeHWAccel,
@@ -134,6 +135,8 @@ describe(MediaService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', {
size: 1440,
format: 'jpeg',
quality: 80,
colorspace: Colorspace.P3,
});
expect(assetMock.save).toHaveBeenCalledWith({
id: 'asset-id',
@@ -148,12 +151,11 @@ describe(MediaService.name, () => {
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
inputOptions: [],
inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [
'-ss 00:00:00.000',
'-frames:v 1',
'-v verbose',
'-vf scale=-2:1440:out_color_matrix=bt601:out_range=pc,format=yuv420p',
'-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
],
twoPass: false,
});
@@ -170,12 +172,11 @@ describe(MediaService.name, () => {
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
inputOptions: [],
inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [
'-ss 00:00:00.000',
'-frames:v 1',
'-v verbose',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt470bg:t=601:m=bt470bg:range=pc,format=yuv420p',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
],
twoPass: false,
});
@@ -209,12 +210,13 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id });
expect(mediaMock.resize).toHaveBeenCalledWith(
'/uploads/user-id/thumbs/path.jpg',
'/uploads/user-id/thumbs/path.webp',
{ format: 'webp', size: 250 },
);
expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: '/uploads/user-id/thumbs/path.webp' });
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', {
format: 'webp',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' });
});
});

78
server/src/domain/media/media.service.ts
View File

@@ -9,7 +9,6 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config
import { SystemConfigCore } from '../system-config/system-config.core';
import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository';
import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
@Injectable()
export class MediaService {
private logger = new Logger(MediaService.name);
@@ -21,9 +20,9 @@ export class MediaService {
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ISystemConfigRepository) systemConfig: ISystemConfigRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
) {
this.configCore = new SystemConfigCore(systemConfig);
this.configCore = new SystemConfigCore(configRepository);
}
async handleQueueGenerateThumbnails(job: IBaseJob) {
@@ -59,38 +58,53 @@ export class MediaService {
return false;
}
const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
this.storageRepository.mkdirSync(resizePath);
const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
const { thumbnail } = await this.configCore.getConfig();
const resizePath = await this.generateThumbnail(asset, 'jpeg');
await this.assetRepository.save({ id: asset.id, resizePath });
return true;
}
async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
let path;
switch (asset.type) {
case AssetType.IMAGE:
await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, {
size: thumbnail.jpegSize,
format: 'jpeg',
});
this.logger.log(`Successfully generated image thumbnail ${asset.id}`);
path = await this.generateImageThumbnail(asset, format);
break;
case AssetType.VIDEO:
const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) {
this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`);
return false;
}
const mainAudioStream = this.getMainStream(audioStreams);
const { ffmpeg } = await this.configCore.getConfig();
const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options);
this.logger.log(`Successfully generated video thumbnail ${asset.id}`);
path = await this.generateVideoThumbnail(asset, format);
break;
default:
throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`);
}
this.logger.log(
`Successfully generated ${format.toUpperCase()} ${asset.type.toLowerCase()} thumbnail for asset ${asset.id}`,
);
return path;
}
await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { thumbnail } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const thumbnailOptions = { format, size, colorspace: thumbnail.colorspace, quality: thumbnail.quality };
const path = this.ensureThumbnailPath(asset, format);
await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions);
return path;
}
return true;
async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { ffmpeg, thumbnail } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) {
this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`);
return;
}
const mainAudioStream = this.getMainStream(audioStreams);
const path = this.ensureThumbnailPath(asset, format);
const config = { ...ffmpeg, targetResolution: size.toString() };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, path, options);
return path;
}
async handleGenerateWebpThumbnail({ id }: IEntityJob) {
@@ -99,12 +113,8 @@ export class MediaService {
return false;
}
const webpPath = asset.resizePath.replace('jpeg', 'webp').replace('jpg', 'webp');
const { thumbnail } = await this.configCore.getConfig();
await this.mediaRepository.resize(asset.resizePath, webpPath, { size: thumbnail.webpSize, format: 'webp' });
const webpPath = await this.generateThumbnail(asset, 'webp');
await this.assetRepository.save({ id: asset.id, webpPath });
return true;
}
@@ -289,4 +299,10 @@ export class MediaService {
return handler;
}
ensureThumbnailPath(asset: AssetEntity, extension: string): string {
const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
this.storageRepository.mkdirSync(folderPath);
return join(folderPath, `${asset.id}.${extension}`);
}
}

11
server/src/domain/media/media.util.ts
View File

@@ -263,8 +263,11 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
}
export class ThumbnailConfig extends BaseConfig {
getBaseInputOptions(): string[] {
return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'];
}
getBaseOutputOptions() {
return ['-ss 00:00:00.000', '-frames:v 1'];
return ['-frames:v 1'];
}
getPresetOptions() {
@@ -277,16 +280,16 @@ export class ThumbnailConfig extends BaseConfig {
getScaling(videoStream: VideoStreamInfo) {
let options = super.getScaling(videoStream);
options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int';
if (!this.shouldToneMap(videoStream)) {
options += ':out_color_matrix=bt601:out_range=pc';
options += ':out_color_matrix=601:out_range=pc';
}
return options;
}
getColors() {
return {
// jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts
primaries: 'bt470bg',
primaries: 'bt709',
transfer: '601',
matrix: 'bt470bg',
};

16
server/src/domain/system-config/dto/system-config-thumbnail.dto.ts
View File

@@ -1,15 +1,29 @@
import { Colorspace } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsInt } from 'class-validator';
import { IsEnum, IsInt, Max, Min } from 'class-validator';
export class SystemConfigThumbnailDto {
@IsInt()
@Min(1)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
webpSize!: number;
@IsInt()
@Min(1)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
jpegSize!: number;
@IsInt()
@Min(1)
@Max(100)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
quality!: number;
@IsEnum(Colorspace)
@ApiProperty({ enumName: 'Colorspace', enum: Colorspace })
colorspace!: Colorspace;
}

3
server/src/domain/system-config/system-config.core.ts
View File

@@ -1,6 +1,7 @@
import {
AudioCodec,
CQMode,
Colorspace,
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
@@ -98,6 +99,8 @@ export const defaults = Object.freeze<SystemConfig>({
thumbnail: {
webpSize: 250,
jpegSize: 1440,
quality: 80,
colorspace: Colorspace.P3,
},
});

3
server/src/domain/system-config/system-config.service.spec.ts
View File

@@ -1,6 +1,7 @@
import {
AudioCodec,
CQMode,
Colorspace,
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
@@ -94,6 +95,8 @@ const updatedConfig = Object.freeze<SystemConfig>({
thumbnail: {
webpSize: 250,
jpegSize: 1440,
quality: 80,
colorspace: Colorspace.P3,
},
});