KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(server): Store generated files (thumbnails, encoded video) in subdirectories (#4112)

Browse Source 
* save thumbnails in subdirectories

* migration job, migrate assets and face thumbnails

* fix tests

* directory depth of two instead of three

* cleanup empty dirs after migration

* clean up empty dirs after migration, migrate people without assetId

* add job card for new migration job

* fix removeEmptyDirs race condition because of missing await

* cleanup empty directories after asset deletion

* move ensurePath to storage core

* rename jobs

* remove unnecessary property of IEntityJob

* use updated person getById, minor refactoring

* ensure that directory cleanup doesn't interfere with migration

* better description for job in ui

* fix remove directories when migration is done

* cleanup empty folders at start of migration

* fix: actually persist concurrency setting

* add comment explaining regex

* chore: cleanup

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Daniel Dietzler
2023-09-25 17:07:21 +02:00
committed by GitHub
parent 07069c3b1e
commit 3053cbd4c8
36 changed files with 310 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
cli/src/api/open-api/api.ts generated
View File

@@ -307,6 +307,12 @@ export interface AllJobStatusResponseDto {
* @memberof AllJobStatusResponseDto
*/
'metadataExtraction': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'migration': JobStatusDto;
/**
*
* @type {JobStatusDto}
@@ -1779,6 +1785,7 @@ export const JobName = {
ClipEncoding: 'clipEncoding',
BackgroundTask: 'backgroundTask',
StorageTemplateMigration: 'storageTemplateMigration',
Migration: 'migration',
Search: 'search',
Sidecar: 'sidecar',
Library: 'library'
@@ -3240,6 +3247,12 @@ export interface SystemConfigJobDto {
* @memberof SystemConfigJobDto
*/
'metadataExtraction': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'migration': JobSettingsDto;
/**
*
* @type {JobSettingsDto}

1
mobile/openapi/doc/AllJobStatusResponseDto.md generated
View File

@@ -12,6 +12,7 @@ Name | Type | Description | Notes
**clipEncoding** | [**JobStatusDto**](JobStatusDto.md) | |
**library_** | [**JobStatusDto**](JobStatusDto.md) | |
**metadataExtraction** | [**JobStatusDto**](JobStatusDto.md) | |
**migration** | [**JobStatusDto**](JobStatusDto.md) | |
**objectTagging** | [**JobStatusDto**](JobStatusDto.md) | |
**recognizeFaces** | [**JobStatusDto**](JobStatusDto.md) | |
**search** | [**JobStatusDto**](JobStatusDto.md) | |

1
mobile/openapi/doc/SystemConfigJobDto.md generated
View File

@@ -12,6 +12,7 @@ Name | Type | Description | Notes
**clipEncoding** | [**JobSettingsDto**](JobSettingsDto.md) | |
**library_** | [**JobSettingsDto**](JobSettingsDto.md) | |
**metadataExtraction** | [**JobSettingsDto**](JobSettingsDto.md) | |
**migration** | [**JobSettingsDto**](JobSettingsDto.md) | |
**objectTagging** | [**JobSettingsDto**](JobSettingsDto.md) | |
**recognizeFaces** | [**JobSettingsDto**](JobSettingsDto.md) | |
**search** | [**JobSettingsDto**](JobSettingsDto.md) | |

10
mobile/openapi/lib/model/all_job_status_response_dto.dart generated
View File

@@ -17,6 +17,7 @@ class AllJobStatusResponseDto {
required this.clipEncoding,
required this.library_,
required this.metadataExtraction,
required this.migration,
required this.objectTagging,
required this.recognizeFaces,
required this.search,
@@ -34,6 +35,8 @@ class AllJobStatusResponseDto {
JobStatusDto metadataExtraction;
JobStatusDto migration;
JobStatusDto objectTagging;
JobStatusDto recognizeFaces;
@@ -54,6 +57,7 @@ class AllJobStatusResponseDto {
other.clipEncoding == clipEncoding &&
other.library_ == library_ &&
other.metadataExtraction == metadataExtraction &&
other.migration == migration &&
other.objectTagging == objectTagging &&
other.recognizeFaces == recognizeFaces &&
other.search == search &&
@@ -69,6 +73,7 @@ class AllJobStatusResponseDto {
(clipEncoding.hashCode) +
(library_.hashCode) +
(metadataExtraction.hashCode) +
(migration.hashCode) +
(objectTagging.hashCode) +
(recognizeFaces.hashCode) +
(search.hashCode) +
@@ -78,7 +83,7 @@ class AllJobStatusResponseDto {
(videoConversion.hashCode);
@override
String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -86,6 +91,7 @@ class AllJobStatusResponseDto {
json[r'clipEncoding'] = this.clipEncoding;
json[r'library'] = this.library_;
json[r'metadataExtraction'] = this.metadataExtraction;
json[r'migration'] = this.migration;
json[r'objectTagging'] = this.objectTagging;
json[r'recognizeFaces'] = this.recognizeFaces;
json[r'search'] = this.search;
@@ -108,6 +114,7 @@ class AllJobStatusResponseDto {
clipEncoding: JobStatusDto.fromJson(json[r'clipEncoding'])!,
library_: JobStatusDto.fromJson(json[r'library'])!,
metadataExtraction: JobStatusDto.fromJson(json[r'metadataExtraction'])!,
migration: JobStatusDto.fromJson(json[r'migration'])!,
objectTagging: JobStatusDto.fromJson(json[r'objectTagging'])!,
recognizeFaces: JobStatusDto.fromJson(json[r'recognizeFaces'])!,
search: JobStatusDto.fromJson(json[r'search'])!,
@@ -166,6 +173,7 @@ class AllJobStatusResponseDto {
'clipEncoding',
'library',
'metadataExtraction',
'migration',
'objectTagging',
'recognizeFaces',
'search',

3
mobile/openapi/lib/model/job_name.dart generated
View File

@@ -31,6 +31,7 @@ class JobName {
static const clipEncoding = JobName._(r'clipEncoding');
static const backgroundTask = JobName._(r'backgroundTask');
static const storageTemplateMigration = JobName._(r'storageTemplateMigration');
static const migration = JobName._(r'migration');
static const search = JobName._(r'search');
static const sidecar = JobName._(r'sidecar');
static const library_ = JobName._(r'library');
@@ -45,6 +46,7 @@ class JobName {
clipEncoding,
backgroundTask,
storageTemplateMigration,
migration,
search,
sidecar,
library_,
@@ -94,6 +96,7 @@ class JobNameTypeTransformer {
case r'clipEncoding': return JobName.clipEncoding;
case r'backgroundTask': return JobName.backgroundTask;
case r'storageTemplateMigration': return JobName.storageTemplateMigration;
case r'migration': return JobName.migration;
case r'search': return JobName.search;
case r'sidecar': return JobName.sidecar;
case r'library': return JobName.library_;

10
mobile/openapi/lib/model/system_config_job_dto.dart generated
View File

@@ -17,6 +17,7 @@ class SystemConfigJobDto {
required this.clipEncoding,
required this.library_,
required this.metadataExtraction,
required this.migration,
required this.objectTagging,
required this.recognizeFaces,
required this.search,
@@ -34,6 +35,8 @@ class SystemConfigJobDto {
JobSettingsDto metadataExtraction;
JobSettingsDto migration;
JobSettingsDto objectTagging;
JobSettingsDto recognizeFaces;
@@ -54,6 +57,7 @@ class SystemConfigJobDto {
other.clipEncoding == clipEncoding &&
other.library_ == library_ &&
other.metadataExtraction == metadataExtraction &&
other.migration == migration &&
other.objectTagging == objectTagging &&
other.recognizeFaces == recognizeFaces &&
other.search == search &&
@@ -69,6 +73,7 @@ class SystemConfigJobDto {
(clipEncoding.hashCode) +
(library_.hashCode) +
(metadataExtraction.hashCode) +
(migration.hashCode) +
(objectTagging.hashCode) +
(recognizeFaces.hashCode) +
(search.hashCode) +
@@ -78,7 +83,7 @@ class SystemConfigJobDto {
(videoConversion.hashCode);
@override
String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -86,6 +91,7 @@ class SystemConfigJobDto {
json[r'clipEncoding'] = this.clipEncoding;
json[r'library'] = this.library_;
json[r'metadataExtraction'] = this.metadataExtraction;
json[r'migration'] = this.migration;
json[r'objectTagging'] = this.objectTagging;
json[r'recognizeFaces'] = this.recognizeFaces;
json[r'search'] = this.search;
@@ -108,6 +114,7 @@ class SystemConfigJobDto {
clipEncoding: JobSettingsDto.fromJson(json[r'clipEncoding'])!,
library_: JobSettingsDto.fromJson(json[r'library'])!,
metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
migration: JobSettingsDto.fromJson(json[r'migration'])!,
objectTagging: JobSettingsDto.fromJson(json[r'objectTagging'])!,
recognizeFaces: JobSettingsDto.fromJson(json[r'recognizeFaces'])!,
search: JobSettingsDto.fromJson(json[r'search'])!,
@@ -166,6 +173,7 @@ class SystemConfigJobDto {
'clipEncoding',
'library',
'metadataExtraction',
'migration',
'objectTagging',
'recognizeFaces',
'search',

5
mobile/openapi/test/all_job_status_response_dto_test.dart generated
View File

@@ -36,6 +36,11 @@ void main() {
// TODO
});
// JobStatusDto migration
test('to test the property `migration`', () async {
// TODO
});
// JobStatusDto objectTagging
test('to test the property `objectTagging`', () async {
// TODO

5
mobile/openapi/test/system_config_job_dto_test.dart generated
View File

@@ -36,6 +36,11 @@ void main() {
// TODO
});
// JobSettingsDto migration
test('to test the property `migration`', () async {
// TODO
});
// JobSettingsDto objectTagging
test('to test the property `objectTagging`', () async {
// TODO

9
server/immich-openapi-specs.json
View File

@@ -5343,6 +5343,9 @@
"metadataExtraction": {
"$ref": "#/components/schemas/JobStatusDto"
},
"migration": {
"$ref": "#/components/schemas/JobStatusDto"
},
"objectTagging": {
"$ref": "#/components/schemas/JobStatusDto"
},
@@ -5372,6 +5375,7 @@
"objectTagging",
"clipEncoding",
"storageTemplateMigration",
"migration",
"backgroundTask",
"search",
"recognizeFaces",
@@ -6535,6 +6539,7 @@
"clipEncoding",
"backgroundTask",
"storageTemplateMigration",
"migration",
"search",
"sidecar",
"library"
@@ -7693,6 +7698,9 @@
"metadataExtraction": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"migration": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"objectTagging": {
"$ref": "#/components/schemas/JobSettingsDto"
},
@@ -7722,6 +7730,7 @@
"objectTagging",
"clipEncoding",
"storageTemplateMigration",
"migration",
"backgroundTask",
"search",
"recognizeFaces",

3
server/src/domain/asset/asset.service.ts
View File

@@ -57,7 +57,7 @@ export interface UploadFile {
export class AssetService {
private logger = new Logger(AssetService.name);
private access: AccessCore;
private storageCore = new StorageCore();
private storageCore: StorageCore;
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@@ -67,6 +67,7 @@ export class AssetService {
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.access = new AccessCore(accessRepository);
this.storageCore = new StorageCore(storageRepository);
}
canUploadFile({ authUser, fieldName, file }: UploadRequest): true {

10
server/src/domain/facial-recognition/facial-recognition.service.spec.ts
View File

@@ -307,14 +307,14 @@ describe(FacialRecognitionService.name, () => {
await sut.handleGenerateFaceThumbnail(face.middle);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/pe/rs');
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 95,
top: 95,
width: 110,
height: 110,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
@@ -323,7 +323,7 @@ describe(FacialRecognitionService.name, () => {
expect(personMock.update).toHaveBeenCalledWith({
faceAssetId: 'asset-1',
id: 'person-1',
thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg',
thumbnailPath: 'upload/thumbs/user-id/pe/rs/person-1.jpeg',
});
});
@@ -338,7 +338,7 @@ describe(FacialRecognitionService.name, () => {
width: 510,
height: 510,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
@@ -357,7 +357,7 @@ describe(FacialRecognitionService.name, () => {
width: 202,
height: 202,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,

23
server/src/domain/facial-recognition/facial-recognition.services.ts
View File

@@ -1,5 +1,4 @@
import { Inject, Logger } from '@nestjs/common';
import { join } from 'path';
import { IAssetRepository, WithoutProperty } from '../asset';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, IFaceThumbnailJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
@@ -13,8 +12,8 @@ import { AssetFaceId, IFaceRepository } from './face.repository';
export class FacialRecognitionService {
private logger = new Logger(FacialRecognitionService.name);
private storageCore = new StorageCore();
private configCore: SystemConfigCore;
private storageCore: StorageCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -28,6 +27,7 @@ export class FacialRecognitionService {
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.configCore = new SystemConfigCore(configRepository);
this.storageCore = new StorageCore(storageRepository);
}
async handleQueueRecognizeFaces({ force }: IBaseJob) {
@@ -117,6 +117,21 @@ export class FacialRecognitionService {
return true;
}
async handlePersonMigration({ id }: IEntityJob) {
const person = await this.personRepository.getById(id);
if (!person) {
return false;
}
const path = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, person.ownerId, `${id}.jpeg`);
if (person.thumbnailPath && person.thumbnailPath !== path) {
await this.storageRepository.moveFile(person.thumbnailPath, path);
await this.personRepository.update({ id, thumbnailPath: path });
}
return true;
}
async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
@@ -132,9 +147,7 @@ export class FacialRecognitionService {
this.logger.verbose(`Cropping face for person: ${personId}`);
const outputFolder = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
const output = join(outputFolder, `${personId}.jpeg`);
this.storageRepository.mkdirSync(outputFolder);
const output = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`);
const { x1, y1, x2, y2 } = boundingBox;

11
server/src/domain/job/job.constants.ts
View File

@@ -7,6 +7,7 @@ export enum QueueName {
CLIP_ENCODING = 'clipEncoding',
BACKGROUND_TASK = 'backgroundTask',
STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
MIGRATION = 'migration',
SEARCH = 'search',
SIDECAR = 'sidecar',
LIBRARY = 'library',
@@ -45,6 +46,11 @@ export enum JobName {
STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single',
SYSTEM_CONFIG_CHANGE = 'system-config-change',
// migration
QUEUE_MIGRATION = 'queue-migration',
MIGRATE_ASSET = 'migrate-asset',
MIGRATE_PERSON = 'migrate-person',
// object tagging
QUEUE_OBJECT_TAGGING = 'queue-object-tagging',
CLASSIFY_IMAGE = 'classify-image',
@@ -119,6 +125,11 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: QueueName.STORAGE_TEMPLATE_MIGRATION,
[JobName.SYSTEM_CONFIG_CHANGE]: QueueName.STORAGE_TEMPLATE_MIGRATION,
// migration
[JobName.QUEUE_MIGRATION]: QueueName.MIGRATION,
[JobName.MIGRATE_ASSET]: QueueName.MIGRATION,
[JobName.MIGRATE_PERSON]: QueueName.MIGRATION,
// object tagging
[JobName.QUEUE_OBJECT_TAGGING]: QueueName.OBJECT_TAGGING,
[JobName.CLASSIFY_IMAGE]: QueueName.OBJECT_TAGGING,

3
server/src/domain/job/job.dto.ts
View File

@@ -68,6 +68,9 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
@ApiProperty({ type: JobStatusDto })
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.MIGRATION]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.BACKGROUND_TASK]!: JobStatusDto;

5
server/src/domain/job/job.repository.ts
View File

@@ -46,6 +46,11 @@ export type JobItem =
| { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob }
| { name: JobName.SYSTEM_CONFIG_CHANGE; data?: IBaseJob }
// Migration
| { name: JobName.QUEUE_MIGRATION; data?: IBaseJob }
| { name: JobName.MIGRATE_ASSET; data?: IEntityJob }
| { name: JobName.MIGRATE_PERSON; data?: IEntityJob }
// Metadata Extraction
| { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
| { name: JobName.METADATA_EXTRACTION; data: IEntityJob }

3
server/src/domain/job/job.service.spec.ts
View File

@@ -94,6 +94,7 @@ describe(JobService.name, () => {
[QueueName.OBJECT_TAGGING]: expectedJobStatus,
[QueueName.SEARCH]: expectedJobStatus,
[QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus,
[QueueName.MIGRATION]: expectedJobStatus,
[QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
@@ -229,6 +230,7 @@ describe(JobService.name, () => {
[QueueName.SIDECAR]: { concurrency: 10 },
[QueueName.LIBRARY]: { concurrency: 10 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
[QueueName.MIGRATION]: { concurrency: 10 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
},
@@ -242,6 +244,7 @@ describe(JobService.name, () => {
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
});

3
server/src/domain/job/job.service.ts
View File

@@ -76,6 +76,9 @@ export class JobService {
case QueueName.STORAGE_TEMPLATE_MIGRATION:
return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
case QueueName.MIGRATION:
return this.jobRepository.queue({ name: JobName.QUEUE_MIGRATION });
case QueueName.OBJECT_TAGGING:
await this.configCore.requireFeature(FeatureFlag.TAG_IMAGE);
return this.jobRepository.queue({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force } });

135
server/src/domain/media/media.service.spec.ts
View File

@@ -202,8 +202,8 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateJpegThumbnail({ id: assetStub.image.id });
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', {
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/as/se/asset-id.jpeg', {
size: 1440,
format: 'jpeg',
quality: 80,
@@ -211,7 +211,7 @@ describe(MediaService.name, () => {
});
expect(assetMock.save).toHaveBeenCalledWith({
id: 'asset-id',
resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg',
});
});
@@ -220,19 +220,23 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id });
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [
'-frames:v 1',
'-v verbose',
'-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
],
twoPass: false,
});
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/thumbs/user-id/as/se/asset-id.jpeg',
{
inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [
'-frames:v 1',
'-v verbose',
'-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
],
twoPass: false,
},
);
expect(assetMock.save).toHaveBeenCalledWith({
id: 'asset-id',
resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg',
});
});
@@ -241,19 +245,23 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id });
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [
'-frames:v 1',
'-v verbose',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
],
twoPass: false,
});
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/thumbs/user-id/as/se/asset-id.jpeg',
{
inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [
'-frames:v 1',
'-v verbose',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
],
twoPass: false,
},
);
expect(assetMock.save).toHaveBeenCalledWith({
id: 'asset-id',
resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg',
});
});
@@ -275,13 +283,16 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id });
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', {
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/as/se/asset-id.webp', {
format: 'webp',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' });
expect(assetMock.save).toHaveBeenCalledWith({
id: 'asset-id',
webpPath: 'upload/thumbs/user-id/as/se/asset-id.webp',
});
});
});
@@ -375,7 +386,7 @@ describe(MediaService.name, () => {
expect(storageMock.mkdirSync).toHaveBeenCalled();
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -416,7 +427,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -442,7 +453,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -471,7 +482,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -498,7 +509,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -525,7 +536,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -552,7 +563,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -603,7 +614,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -635,7 +646,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -664,7 +675,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -695,7 +706,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -728,7 +739,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -760,7 +771,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -791,7 +802,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -821,7 +832,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -851,7 +862,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -881,7 +892,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -914,7 +925,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -976,7 +987,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
@@ -1014,7 +1025,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
@@ -1048,7 +1059,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
@@ -1083,7 +1094,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
@@ -1114,7 +1125,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [
@@ -1150,7 +1161,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [
@@ -1186,7 +1197,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [
@@ -1219,7 +1230,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [
@@ -1263,7 +1274,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [
@@ -1295,7 +1306,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [
@@ -1329,7 +1340,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [
@@ -1359,7 +1370,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'],
outputOptions: [
@@ -1385,7 +1396,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'],
outputOptions: [
@@ -1418,7 +1429,7 @@ describe(MediaService.name, () => {
expect(mediaMock.transcode).toHaveBeenCalledTimes(2);
expect(mediaMock.transcode).toHaveBeenLastCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -1455,7 +1466,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -1482,7 +1493,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
@@ -1509,7 +1520,7 @@ describe(MediaService.name, () => {
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/asset-id.mp4',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [

74
server/src/domain/media/media.service.ts
View File

@@ -1,9 +1,8 @@
import { AssetEntity, AssetType, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common';
import { join } from 'path';
import { IAssetRepository, WithoutProperty } from '../asset';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
import { IPersonRepository } from '../person';
import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config';
@@ -14,8 +13,8 @@ import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIC
@Injectable()
export class MediaService {
private logger = new Logger(MediaService.name);
private storageCore = new StorageCore();
private configCore: SystemConfigCore;
private storageCore: StorageCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -26,11 +25,10 @@ export class MediaService {
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
) {
this.configCore = new SystemConfigCore(configRepository);
this.storageCore = new StorageCore(this.storageRepository);
}
async handleQueueGenerateThumbnails(job: IBaseJob) {
const { force } = job;
async handleQueueGenerateThumbnails({ force }: IBaseJob) {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
@@ -81,6 +79,58 @@ export class MediaService {
return true;
}
async handleQueueMigration() {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination),
);
const { active, waiting } = await this.jobRepository.getJobCounts(QueueName.MIGRATION);
if (active === 1 && waiting === 0) {
await this.storageCore.removeEmptyDirs(StorageFolder.THUMBNAILS);
await this.storageCore.removeEmptyDirs(StorageFolder.ENCODED_VIDEO);
}
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } });
}
}
const people = await this.personRepository.getAll();
for (const person of people) {
await this.jobRepository.queue({ name: JobName.MIGRATE_PERSON, data: { id: person.id } });
}
return true;
}
async handleAssetMigration({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset) {
return false;
}
const resizePath = this.ensureThumbnailPath(asset, 'jpeg');
const webpPath = this.ensureThumbnailPath(asset, 'webp');
const encodedVideoPath = this.ensureEncodedVideoPath(asset, 'mp4');
if (asset.resizePath && asset.resizePath !== resizePath) {
await this.storageRepository.moveFile(asset.resizePath, resizePath);
await this.assetRepository.save({ id: asset.id, resizePath });
}
if (asset.webpPath && asset.webpPath !== webpPath) {
await this.storageRepository.moveFile(asset.webpPath, webpPath);
await this.assetRepository.save({ id: asset.id, webpPath });
}
if (asset.encodedVideoPath && asset.encodedVideoPath !== encodedVideoPath) {
await this.storageRepository.moveFile(asset.encodedVideoPath, encodedVideoPath);
await this.assetRepository.save({ id: asset.id, encodedVideoPath });
}
return true;
}
async handleGenerateJpegThumbnail({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset) {
@@ -184,9 +234,7 @@ export class MediaService {
}
const input = asset.originalPath;
const outputFolder = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId);
const output = join(outputFolder, `${asset.id}.mp4`);
this.storageRepository.mkdirSync(outputFolder);
const output = this.ensureEncodedVideoPath(asset, 'mp4');
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);
const mainVideoStream = this.getMainStream(videoStreams);
@@ -330,8 +378,10 @@ export class MediaService {
}
ensureThumbnailPath(asset: AssetEntity, extension: string): string {
const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
this.storageRepository.mkdirSync(folderPath);
return join(folderPath, `${asset.id}.${extension}`);
return this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.${extension}`);
}
ensureEncodedVideoPath(asset: AssetEntity, extension: string): string {
return this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.${extension}`);
}
}

4
server/src/domain/person/person.service.spec.ts
View File

@@ -37,10 +37,10 @@ describe(PersonService.name, () => {
beforeEach(async () => {
accessMock = newAccessRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
configMock = newSystemConfigRepositoryMock();
jobMock = newJobRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
sut = new PersonService(accessMock, personMock, configMock, storageMock, jobMock);
});

3
server/src/domain/server-info/server-info.service.ts
View File

@@ -16,8 +16,8 @@ import {
@Injectable()
export class ServerInfoService {
private storageCore = new StorageCore();
private configCore: SystemConfigCore;
private storageCore: StorageCore;
constructor(
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@@ -25,6 +25,7 @@ export class ServerInfoService {
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.configCore = new SystemConfigCore(configRepository);
this.storageCore = new StorageCore(storageRepository);
}
async getInfo(): Promise<ServerInfoResponseDto> {

3
server/src/domain/storage-template/storage-template.service.ts
View File

@@ -30,7 +30,7 @@ export interface MoveAssetMetadata {
export class StorageTemplateService {
private logger = new Logger(StorageTemplateService.name);
private configCore: SystemConfigCore;
private storageCore = new StorageCore();
private storageCore: StorageCore;
private storageTemplate: HandlebarsTemplateDelegate<any>;
constructor(
@@ -44,6 +44,7 @@ export class StorageTemplateService {
this.configCore = new SystemConfigCore(configRepository);
this.configCore.addValidator((config) => this.validate(config));
this.configCore.config$.subscribe((config) => this.onConfig(config));
this.storageCore = new StorageCore(storageRepository);
}
async handleMigrationSingle({ id }: IEntityJob) {

21
server/src/domain/storage/storage.core.ts
View File

@@ -1,5 +1,6 @@
import { join } from 'node:path';
import { APP_MEDIA_LOCATION } from '../domain.constant';
import { IStorageRepository } from './storage.repository';
export enum StorageFolder {
ENCODED_VIDEO = 'encoded-video',
@@ -10,6 +11,8 @@ export enum StorageFolder {
}
export class StorageCore {
constructor(private repository: IStorageRepository) {}
getFolderLocation(
folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
userId: string,
@@ -24,4 +27,22 @@ export class StorageCore {
getBaseFolder(folder: StorageFolder) {
return join(APP_MEDIA_LOCATION, folder);
}
ensurePath(
folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
ownerId: string,
fileName: string,
): string {
const folderPath = join(
this.getFolderLocation(folder, ownerId),
fileName.substring(0, 2),
fileName.substring(2, 4),
);
this.repository.mkdirSync(folderPath);
return join(folderPath, fileName);
}
removeEmptyDirs(folder: StorageFolder) {
return this.repository.removeEmptyDirs(this.getBaseFolder(folder));
}
}

2
server/src/domain/storage/storage.repository.ts
View File

@@ -26,7 +26,7 @@ export interface IStorageRepository {
createReadStream(filepath: string, mimeType?: string | null): Promise<ImmichReadStream>;
unlink(filepath: string): Promise<void>;
unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
removeEmptyDirs(folder: string): Promise<void>;
removeEmptyDirs(folder: string, self?: boolean): Promise<void>;
moveFile(source: string, target: string): Promise<void>;
checkFileExists(filepath: string, mode?: number): Promise<boolean>;
mkdirSync(filepath: string): void;

6
server/src/domain/storage/storage.service.ts
View File

@@ -6,9 +6,11 @@ import { IStorageRepository } from './storage.repository';
@Injectable()
export class StorageService {
private logger = new Logger(StorageService.name);
private storageCore = new StorageCore();
private storageCore: StorageCore;
constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {}
constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {
this.storageCore = new StorageCore(storageRepository);
}
init() {
const libraryBase = this.storageCore.getBaseFolder(StorageFolder.LIBRARY);

6
server/src/domain/system-config/dto/system-config-job.dto.ts
View File

@@ -47,6 +47,12 @@ export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
@Type(() => JobSettingsDto)
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.MIGRATION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()

1
server/src/domain/system-config/system-config.core.ts
View File

@@ -53,6 +53,7 @@ export const defaults = Object.freeze<SystemConfig>({
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.LIBRARY]: { concurrency: 1 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},

1
server/src/domain/system-config/system-config.service.spec.ts
View File

@@ -33,6 +33,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.LIBRARY]: { concurrency: 1 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},

3
server/src/domain/user/user.service.ts
View File

@@ -25,8 +25,8 @@ import { IUserRepository } from './user.repository';
@Injectable()
export class UserService {
private logger = new Logger(UserService.name);
private storageCore: StorageCore;
private userCore: UserCore;
private storageCore = new StorageCore();
constructor(
@Inject(IUserRepository) private userRepository: IUserRepository,
@@ -37,6 +37,7 @@ export class UserService {
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.storageCore = new StorageCore(storageRepository);
this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
}

1
server/src/infra/entities/system-config.entity.ts
View File

@@ -43,6 +43,7 @@ export enum SystemConfigKey {
JOB_SEARCH_CONCURRENCY = 'job.search.concurrency',
JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency',
JOB_LIBRARY_CONCURRENCY = 'job.library.concurrency',
JOB_MIGRATION_CONCURRENCY = 'job.migration.concurrency',
MACHINE_LEARNING_ENABLED = 'machineLearning.enabled',
MACHINE_LEARNING_URL = 'machineLearning.url',

8
server/src/infra/repositories/filesystem.provider.ts
View File

@@ -66,11 +66,7 @@ export class FilesystemProvider implements IStorageRepository {
await fs.rm(folder, options);
}
async removeEmptyDirs(directory: string) {
this._removeEmptyDirs(directory, false);
}
private async _removeEmptyDirs(directory: string, self: boolean) {
async removeEmptyDirs(directory: string, self: boolean = false) {
// lstat does not follow symlinks (in contrast to stat)
const stats = await fs.lstat(directory);
if (!stats.isDirectory()) {
@@ -78,7 +74,7 @@ export class FilesystemProvider implements IStorageRepository {
}
const files = await fs.readdir(directory);
await Promise.all(files.map((file) => this._removeEmptyDirs(path.join(directory, file), true)));
await Promise.all(files.map((file) => this.removeEmptyDirs(path.join(directory, file), true)));
if (self) {
const updated = await fs.readdir(directory);

3
server/src/microservices/app.service.ts
View File

@@ -63,6 +63,9 @@ export class AppService {
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
[JobName.QUEUE_MIGRATION]: () => this.mediaService.handleQueueMigration(),
[JobName.MIGRATE_ASSET]: (data) => this.mediaService.handleAssetMigration(data),
[JobName.MIGRATE_PERSON]: (data) => this.facialRecognitionService.handlePersonMigration(data),
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),

3
server/src/microservices/processors/metadata-extraction.processor.ts
View File

@@ -50,7 +50,7 @@ const validate = <T>(value: T): T | null => (typeof value === 'string' ? null :
export class MetadataExtractionProcessor {
private logger = new Logger(MetadataExtractionProcessor.name);
private reverseGeocodingEnabled: boolean;
private storageCore = new StorageCore();
private storageCore: StorageCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -63,6 +63,7 @@ export class MetadataExtractionProcessor {
configService: ConfigService,
) {
this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING');
this.storageCore = new StorageCore(storageRepository);
}
async init(deleteCache = false) {

1
web/src/api/api.ts
View File

@@ -131,6 +131,7 @@ export class ImmichApi {
[JobName.RecognizeFaces]: 'Recognize Faces',
[JobName.VideoConversion]: 'Transcode Videos',
[JobName.StorageTemplateMigration]: 'Storage Template Migration',
[JobName.Migration]: 'Migration',
[JobName.BackgroundTask]: 'Background Tasks',
[JobName.Search]: 'Search',
[JobName.Library]: 'Library',

13
web/src/api/open-api/api.ts generated
View File

@@ -307,6 +307,12 @@ export interface AllJobStatusResponseDto {
* @memberof AllJobStatusResponseDto
*/
'metadataExtraction': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'migration': JobStatusDto;
/**
*
* @type {JobStatusDto}
@@ -1779,6 +1785,7 @@ export const JobName = {
ClipEncoding: 'clipEncoding',
BackgroundTask: 'backgroundTask',
StorageTemplateMigration: 'storageTemplateMigration',
Migration: 'migration',
Search: 'search',
Sidecar: 'sidecar',
Library: 'library'
@@ -3240,6 +3247,12 @@ export interface SystemConfigJobDto {
* @memberof SystemConfigJobDto
*/
'metadataExtraction': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'migration': JobSettingsDto;
/**
*
* @type {JobSettingsDto}

6
web/src/lib/components/admin-page/jobs/jobs-panel.svelte
View File

@@ -110,6 +110,12 @@
allowForceCommand: false,
component: StorageMigrationDescription,
},
[JobName.Migration]: {
icon: FolderMove,
title: api.getJobName(JobName.Migration),
subtitle: 'Migrate thumbnails for assets and faces to the latest folder structure',
allowForceCommand: false,
},
};
$: jobList = Object.entries(jobDetails) as [JobName, JobDetails][];