refactor(server): job handlers (#2572)

* refactor(server): job handlers * chore: remove comment * chore: add comments for
2026-01-07 09:45:50 +00:00 · 2023-05-26 15:43:24 -04:00
parent d6756f3d81
commit 1c2d83e2c7
33 changed files with 807 additions and 1082 deletions
--- a/server/libs/domain/src/job/job.constants.ts
+++ b/server/libs/domain/src/job/job.constants.ts
@@ -19,9 +19,6 @@ export enum JobCommand {
 }

 export enum JobName {
-  // upload
-  ASSET_UPLOADED = 'asset-uploaded',
-
  // conversion
  QUEUE_VIDEO_CONVERSION = 'queue-video-conversion',
  VIDEO_CONVERSION = 'video-conversion',
@@ -33,8 +30,7 @@ export enum JobName {

  // metadata
  QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction',
-  EXIF_EXTRACTION = 'exif-extraction',
-  EXTRACT_VIDEO_METADATA = 'extract-video-metadata',
+  METADATA_EXTRACTION = 'metadata-extraction',

  // user deletion
  USER_DELETION = 'user-deletion',
@@ -84,7 +80,6 @@ export const JOBS_ASSET_PAGINATION_SIZE = 1000;

 export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
  // misc
-  [JobName.ASSET_UPLOADED]: QueueName.BACKGROUND_TASK,
  [JobName.USER_DELETE_CHECK]: QueueName.BACKGROUND_TASK,
  [JobName.USER_DELETION]: QueueName.BACKGROUND_TASK,
  [JobName.DELETE_FILES]: QueueName.BACKGROUND_TASK,
@@ -101,8 +96,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {

  // metadata
  [JobName.QUEUE_METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION,
-  [JobName.EXIF_EXTRACTION]: QueueName.METADATA_EXTRACTION,
-  [JobName.EXTRACT_VIDEO_METADATA]: QueueName.METADATA_EXTRACTION,
+  [JobName.METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION,

  // storage template
  [JobName.STORAGE_TEMPLATE_MIGRATION]: QueueName.STORAGE_TEMPLATE_MIGRATION,
--- a/server/libs/domain/src/job/job.interface.ts
+++ b/server/libs/domain/src/job/job.interface.ts
@@ -1,18 +1,9 @@
-import { AlbumEntity, AssetEntity, UserEntity } from '@app/infra/entities';
 import { BoundingBox } from '../smart-info';

 export interface IBaseJob {
  force?: boolean;
 }

-export interface IAlbumJob extends IBaseJob {
-  album: AlbumEntity;
-}
-
-export interface IAssetJob extends IBaseJob {
-  asset: AssetEntity;
-}
-
 export interface IAssetFaceJob extends IBaseJob {
  assetId: string;
  personId: string;
@@ -26,6 +17,10 @@ export interface IFaceThumbnailJob extends IAssetFaceJob {
  personId: string;
 }

+export interface IEntityJob extends IBaseJob {
+  id: string;
+}
+
 export interface IBulkEntityJob extends IBaseJob {
  ids: string[];
 }
@@ -33,7 +28,3 @@ export interface IBulkEntityJob extends IBaseJob {
 export interface IDeleteFilesJob extends IBaseJob {
  files: Array<string | null | undefined>;
 }
-
-export interface IUserDeletionJob extends IBaseJob {
-  user: UserEntity;
-}
--- a/server/libs/domain/src/job/job.repository.ts
+++ b/server/libs/domain/src/job/job.repository.ts
@@ -1,12 +1,11 @@
 import { JobName, QueueName } from './job.constants';
 import {
  IAssetFaceJob,
-  IAssetJob,
  IBaseJob,
  IBulkEntityJob,
  IDeleteFilesJob,
+  IEntityJob,
  IFaceThumbnailJob,
-  IUserDeletionJob,
 } from './job.interface';

 export interface JobCounts {
@@ -24,50 +23,46 @@ export interface QueueStatus {
 }

 export type JobItem =
-  // Asset Upload
-  | { name: JobName.ASSET_UPLOADED; data: IAssetJob }
-
  // Transcoding
  | { name: JobName.QUEUE_VIDEO_CONVERSION; data: IBaseJob }
-  | { name: JobName.VIDEO_CONVERSION; data: IAssetJob }
+  | { name: JobName.VIDEO_CONVERSION; data: IEntityJob }

  // Thumbnails
  | { name: JobName.QUEUE_GENERATE_THUMBNAILS; data: IBaseJob }
-  | { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IAssetJob }
-  | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IAssetJob }
+  | { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IEntityJob }
+  | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob }

  // User Deletion
  | { name: JobName.USER_DELETE_CHECK }
-  | { name: JobName.USER_DELETION; data: IUserDeletionJob }
+  | { name: JobName.USER_DELETION; data: IEntityJob }

  // Storage Template
  | { name: JobName.STORAGE_TEMPLATE_MIGRATION }
-  | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IAssetJob }
+  | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob }
  | { name: JobName.SYSTEM_CONFIG_CHANGE }

  // Metadata Extraction
  | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
-  | { name: JobName.EXIF_EXTRACTION; data: IAssetJob }
-  | { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetJob }
+  | { name: JobName.METADATA_EXTRACTION; data: IEntityJob }

  // Sidecar Scanning
  | { name: JobName.QUEUE_SIDECAR; data: IBaseJob }
-  | { name: JobName.SIDECAR_DISCOVERY; data: IAssetJob }
-  | { name: JobName.SIDECAR_SYNC; data: IAssetJob }
+  | { name: JobName.SIDECAR_DISCOVERY; data: IEntityJob }
+  | { name: JobName.SIDECAR_SYNC; data: IEntityJob }

  // Object Tagging
  | { name: JobName.QUEUE_OBJECT_TAGGING; data: IBaseJob }
-  | { name: JobName.DETECT_OBJECTS; data: IAssetJob }
-  | { name: JobName.CLASSIFY_IMAGE; data: IAssetJob }
+  | { name: JobName.DETECT_OBJECTS; data: IEntityJob }
+  | { name: JobName.CLASSIFY_IMAGE; data: IEntityJob }

  // Recognize Faces
  | { name: JobName.QUEUE_RECOGNIZE_FACES; data: IBaseJob }
-  | { name: JobName.RECOGNIZE_FACES; data: IAssetJob }
+  | { name: JobName.RECOGNIZE_FACES; data: IEntityJob }
  | { name: JobName.GENERATE_FACE_THUMBNAIL; data: IFaceThumbnailJob }

  // Clip Embedding
  | { name: JobName.QUEUE_ENCODE_CLIP; data: IBaseJob }
-  | { name: JobName.ENCODE_CLIP; data: IAssetJob }
+  | { name: JobName.ENCODE_CLIP; data: IEntityJob }

  // Filesystem
  | { name: JobName.DELETE_FILES; data: IDeleteFilesJob }
--- a/server/libs/domain/src/job/job.service.spec.ts
+++ b/server/libs/domain/src/job/job.service.spec.ts
@@ -1,14 +1,20 @@
 import { BadRequestException } from '@nestjs/common';
-import { newJobRepositoryMock } from '../../test';
+import { newAssetRepositoryMock, newCommunicationRepositoryMock, newJobRepositoryMock } from '../../test';
+import { IAssetRepository } from '../asset';
+import { ICommunicationRepository } from '../communication';
 import { IJobRepository, JobCommand, JobName, JobService, QueueName } from '../job';

 describe(JobService.name, () => {
  let sut: JobService;
+  let assetMock: jest.Mocked<IAssetRepository>;
+  let communicationMock: jest.Mocked<ICommunicationRepository>;
  let jobMock: jest.Mocked<IJobRepository>;

  beforeEach(async () => {
+    assetMock = newAssetRepositoryMock();
+    communicationMock = newCommunicationRepositoryMock();
    jobMock = newJobRepositoryMock();
-    sut = new JobService(jobMock);
+    sut = new JobService(assetMock, communicationMock, jobMock);
  });

  it('should work', () => {
--- a/server/libs/domain/src/job/job.service.ts
+++ b/server/libs/domain/src/job/job.service.ts
@@ -1,21 +1,21 @@
 import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
+import { IAssetRepository, mapAsset } from '../asset';
+import { CommunicationEvent, ICommunicationRepository } from '../communication';
 import { assertMachineLearningEnabled } from '../domain.constant';
 import { JobCommandDto } from './dto';
 import { JobCommand, JobName, QueueName } from './job.constants';
-import { IJobRepository } from './job.repository';
+import { IJobRepository, JobItem } from './job.repository';
 import { AllJobStatusResponseDto, JobStatusDto } from './response-dto';

@Injectable()
 export class JobService {
  private logger = new Logger(JobService.name);

-  constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
-
-  async handleNightlyJobs() {
-    await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK });
-    await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
-    await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
-  }
+  constructor(
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
+    @Inject(IJobRepository) private jobRepository: IJobRepository,
+  ) {}

  handleCommand(queueName: QueueName, dto: JobCommandDto): Promise<void> {
    this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`);
@@ -89,4 +89,51 @@ export class JobService {
        throw new BadRequestException(`Invalid job name: ${name}`);
    }
  }
+
+  async handleNightlyJobs() {
+    await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK });
+    await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
+    await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
+  }
+
+  /**
+   * Queue follow up jobs
+   */
+  async onDone(item: JobItem) {
+    switch (item.name) {
+      case JobName.SIDECAR_SYNC:
+      case JobName.SIDECAR_DISCOVERY:
+        await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: item.data.id } });
+        break;
+
+      case JobName.METADATA_EXTRACTION:
+        await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: item.data });
+        break;
+
+      case JobName.GENERATE_JPEG_THUMBNAIL: {
+        await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data });
+        await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: item.data });
+        await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: item.data });
+        await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: item.data });
+        await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: item.data });
+
+        const [asset] = await this.assetRepository.getByIds([item.data.id]);
+        if (asset) {
+          this.communicationRepository.send(CommunicationEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
+        }
+        break;
+      }
+    }
+
+    // In addition to the above jobs, all of these should queue `SEARCH_INDEX_ASSET`
+    switch (item.name) {
+      case JobName.CLASSIFY_IMAGE:
+      case JobName.DETECT_OBJECTS:
+      case JobName.ENCODE_CLIP:
+      case JobName.RECOGNIZE_FACES:
+      case JobName.METADATA_EXTRACTION:
+        await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [item.data.id] } });
+        break;
+    }
+  }
 }