KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-12-07 19:59:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): jobs and processors (#1787)

Browse Source 
* refactor: jobs and processors

* refactor: storage migration processor

* fix: tests

* fix: code warning

* chore: ignore coverage from infra

* fix: sync move asset logic between job core and asset core

* refactor: move error handling inside of catch

* refactor(server): job core into dedicated service calls

* refactor: smart info

* fix: tests

* chore: smart info tests

* refactor: use asset repository

* refactor: thumbnail processor

* chore: coverage reqs
This commit is contained in:
Jason Rasmussen
2023-02-25 09:12:03 -05:00
committed by GitHub
parent 71d8567f18
commit 6c7679714b
108 changed files with 1645 additions and 1072 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
server/libs/domain/src/job/index.ts
View File

@@ -1,4 +1,3 @@
export * from './interfaces';
export * from './job.constants';
export * from './job.interface';
export * from './job.repository';
export * from './job.service';

13
server/libs/domain/src/job/interfaces/asset-uploaded.interface.ts
View File

@@ -1,13 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IAssetUploadedJob {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
/**
* Original file name
*/
fileName: string;
}

5
server/libs/domain/src/job/interfaces/background-task.interface.ts
View File

@@ -1,5 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IDeleteFileOnDiskJob {
assets: AssetEntity[];
}

7
server/libs/domain/src/job/interfaces/index.ts
View File

@@ -1,7 +0,0 @@
export * from './asset-uploaded.interface';
export * from './background-task.interface';
export * from './machine-learning.interface';
export * from './metadata-extraction.interface';
export * from './thumbnail-generation.interface';
export * from './user-deletion.interface';
export * from './video-transcode.interface';

8
server/libs/domain/src/job/interfaces/machine-learning.interface.ts
View File

@@ -1,8 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IMachineLearningJob {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}

36
server/libs/domain/src/job/interfaces/metadata-extraction.interface.ts
View File

@@ -1,36 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IExifExtractionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
/**
* Original file name
*/
fileName: string;
}
export interface IVideoLengthExtractionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
/**
* Original file name
*/
fileName: string;
}
export interface IReverseGeocodingProcessor {
assetId: string;
latitude: number;
longitude: number;
}
export type IMetadataExtractionJob =
| IExifExtractionProcessor
| IVideoLengthExtractionProcessor
| IReverseGeocodingProcessor;

17
server/libs/domain/src/job/interfaces/thumbnail-generation.interface.ts
View File

@@ -1,17 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface JpegGeneratorProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export interface WebpGeneratorProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export type IThumbnailGenerationJob = JpegGeneratorProcessor | WebpGeneratorProcessor;

8
server/libs/domain/src/job/interfaces/user-deletion.interface.ts
View File

@@ -1,8 +0,0 @@
import { UserEntity } from '@app/infra/db/entities';
export interface IUserDeletionJob {
/**
* The user entity that was saved in the database
*/
user: UserEntity;
}

10
server/libs/domain/src/job/interfaces/video-transcode.interface.ts
View File

@@ -1,10 +0,0 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IVideoConversionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export type IVideoTranscodeJob = IVideoConversionProcessor;

11
server/libs/domain/src/job/job.constants.ts
View File

@@ -2,11 +2,9 @@ export enum QueueName {
THUMBNAIL_GENERATION = 'thumbnail-generation-queue',
METADATA_EXTRACTION = 'metadata-extraction-queue',
VIDEO_CONVERSION = 'video-conversion-queue',
ASSET_UPLOADED = 'asset-uploaded-queue',
MACHINE_LEARNING = 'machine-learning-queue',
USER_DELETION = 'user-deletion-queue',
CONFIG = 'config-queue',
BACKGROUND_TASK = 'background-task',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration-queue',
}
export enum JobName {
@@ -18,9 +16,10 @@ export enum JobName {
EXTRACT_VIDEO_METADATA = 'extract-video-metadata',
REVERSE_GEOCODING = 'reverse-geocoding',
USER_DELETION = 'user-deletion',
TEMPLATE_MIGRATION = 'template-migration',
CONFIG_CHANGE = 'config-change',
USER_DELETE_CHECK = 'user-delete-check',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
SYSTEM_CONFIG_CHANGE = 'system-config-change',
OBJECT_DETECTION = 'detect-object',
IMAGE_TAGGING = 'tag-image',
DELETE_FILE_ON_DISK = 'delete-file-on-disk',
DELETE_FILES = 'delete-files',
}

26
server/libs/domain/src/job/job.interface.ts Normal file
View File

@@ -0,0 +1,26 @@
import { AssetEntity, UserEntity } from '@app/infra/db/entities';
export interface IAssetJob {
asset: AssetEntity;
}
export interface IAssetUploadedJob {
asset: AssetEntity;
fileName: string;
}
export interface IDeleteFilesJob {
files: Array<string | null | undefined>;
}
export interface IUserDeletionJob {
user: UserEntity;
}
export interface IReverseGeocodingJob {
assetId: string;
latitude: number;
longitude: number;
}
export type IMetadataExtractionJob = IAssetUploadedJob | IReverseGeocodingJob;

42
server/libs/domain/src/job/job.repository.ts
View File

@@ -1,16 +1,5 @@
import {
IAssetUploadedJob,
IDeleteFileOnDiskJob,
IExifExtractionProcessor,
IMachineLearningJob,
IVideoConversionProcessor,
IReverseGeocodingProcessor,
IUserDeletionJob,
IVideoLengthExtractionProcessor,
JpegGeneratorProcessor,
WebpGeneratorProcessor,
} from './interfaces';
import { JobName, QueueName } from './job.constants';
import { IAssetJob, IAssetUploadedJob, IDeleteFilesJob, IReverseGeocodingJob, IUserDeletionJob } from './job.interface';
export interface JobCounts {
active: number;
@@ -20,30 +9,27 @@ export interface JobCounts {
waiting: number;
}
export interface Job<T> {
data: T;
}
export type JobItem =
| { name: JobName.ASSET_UPLOADED; data: IAssetUploadedJob }
| { name: JobName.VIDEO_CONVERSION; data: IVideoConversionProcessor }
| { name: JobName.GENERATE_JPEG_THUMBNAIL; data: JpegGeneratorProcessor }
| { name: JobName.GENERATE_WEBP_THUMBNAIL; data: WebpGeneratorProcessor }
| { name: JobName.EXIF_EXTRACTION; data: IExifExtractionProcessor }
| { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingProcessor }
| { name: JobName.VIDEO_CONVERSION; data: IAssetJob }
| { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IAssetJob }
| { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IAssetJob }
| { name: JobName.EXIF_EXTRACTION; data: IAssetUploadedJob }
| { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingJob }
| { name: JobName.USER_DELETE_CHECK }
| { name: JobName.USER_DELETION; data: IUserDeletionJob }
| { name: JobName.TEMPLATE_MIGRATION }
| { name: JobName.CONFIG_CHANGE }
| { name: JobName.EXTRACT_VIDEO_METADATA; data: IVideoLengthExtractionProcessor }
| { name: JobName.OBJECT_DETECTION; data: IMachineLearningJob }
| { name: JobName.IMAGE_TAGGING; data: IMachineLearningJob }
| { name: JobName.DELETE_FILE_ON_DISK; data: IDeleteFileOnDiskJob };
| { name: JobName.STORAGE_TEMPLATE_MIGRATION }
| { name: JobName.SYSTEM_CONFIG_CHANGE }
| { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetUploadedJob }
| { name: JobName.OBJECT_DETECTION; data: IAssetJob }
| { name: JobName.IMAGE_TAGGING; data: IAssetJob }
| { name: JobName.DELETE_FILES; data: IDeleteFilesJob };
export const IJobRepository = 'IJobRepository';
export interface IJobRepository {
queue(item: JobItem): Promise<void>;
empty(name: QueueName): Promise<void>;
add(item: JobItem): Promise<void>;
isActive(name: QueueName): Promise<boolean>;
getJobCounts(name: QueueName): Promise<JobCounts>;
}

54
server/libs/domain/src/job/job.service.spec.ts
View File

@@ -1,54 +0,0 @@
import { AssetEntity, AssetType } from '@app/infra/db/entities';
import { newJobRepositoryMock } from '../../test';
import { IAssetUploadedJob } from './interfaces';
import { JobName } from './job.constants';
import { IJobRepository, Job } from './job.repository';
import { JobService } from './job.service';
const jobStub = {
upload: {
video: Object.freeze<Job<IAssetUploadedJob>>({
data: { asset: { type: AssetType.VIDEO } as AssetEntity, fileName: 'video.mp4' },
}),
image: Object.freeze<Job<IAssetUploadedJob>>({
data: { asset: { type: AssetType.IMAGE } as AssetEntity, fileName: 'image.jpg' },
}),
},
};
describe(JobService.name, () => {
let sut: JobService;
let jobMock: jest.Mocked<IJobRepository>;
it('should work', () => {
expect(sut).toBeDefined();
});
beforeEach(async () => {
jobMock = newJobRepositoryMock();
sut = new JobService(jobMock);
});
describe('handleUploadedAsset', () => {
it('should process a video', async () => {
await expect(sut.handleUploadedAsset(jobStub.upload.video)).resolves.toBeUndefined();
expect(jobMock.add).toHaveBeenCalledTimes(3);
expect(jobMock.add.mock.calls).toEqual([
[{ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset: { type: AssetType.VIDEO } } }],
[{ name: JobName.VIDEO_CONVERSION, data: { asset: { type: AssetType.VIDEO } } }],
[{ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset: { type: AssetType.VIDEO }, fileName: 'video.mp4' } }],
]);
});
it('should process an image', async () => {
await sut.handleUploadedAsset(jobStub.upload.image);
expect(jobMock.add).toHaveBeenCalledTimes(2);
expect(jobMock.add.mock.calls).toEqual([
[{ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset: { type: AssetType.IMAGE } } }],
[{ name: JobName.EXIF_EXTRACTION, data: { asset: { type: AssetType.IMAGE }, fileName: 'image.jpg' } }],
]);
});
});
});

17
server/libs/domain/src/job/job.service.ts
View File

@@ -1,17 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { IAssetUploadedJob } from './interfaces';
import { JobUploadCore } from './job.upload.core';
import { IJobRepository, Job } from './job.repository';
@Injectable()
export class JobService {
private uploadCore: JobUploadCore;
constructor(@Inject(IJobRepository) repository: IJobRepository) {
this.uploadCore = new JobUploadCore(repository);
}
async handleUploadedAsset(job: Job<IAssetUploadedJob>) {
await this.uploadCore.handleAsset(job);
}
}

32
server/libs/domain/src/job/job.upload.core.ts
View File

@@ -1,32 +0,0 @@
import { AssetType } from '@app/infra/db/entities';
import { IAssetUploadedJob } from './interfaces';
import { JobName } from './job.constants';
import { IJobRepository, Job } from './job.repository';
export class JobUploadCore {
constructor(private repository: IJobRepository) {}
/**
* Post processing uploaded asset to perform the following function
* 1. Generate JPEG Thumbnail
* 2. Generate Webp Thumbnail
* 3. EXIF extractor
* 4. Reverse Geocoding
*
* @param job asset-uploaded
*/
async handleAsset(job: Job<IAssetUploadedJob>) {
const { asset, fileName } = job.data;
await this.repository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
// Video Conversion
if (asset.type == AssetType.VIDEO) {
await this.repository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
await this.repository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName } });
} else {
// Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet
await this.repository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName } });
}
}
}