feat(server): dynamic job concurrency (#2622)

* feat(server): dynamic job concurrency

* styling and add setting info to top of the job list

* regenerate api

* remove DETECT_OBJECT job

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/libs/infra/src/entities/system-config.entity.ts

@@ -1,7 +1,8 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm';
+import { QueueName } from '../../../domain/src';
 
 @Entity('system_config')
-export class SystemConfigEntity<T = string | boolean | number> {
+export class SystemConfigEntity<T = SystemConfigValue> {
   @PrimaryColumn()
   key!: SystemConfigKey;
 
@@ -9,7 +10,7 @@ export class SystemConfigEntity<T = string | boolean | number> {
   value!: T;
 }
 
-export type SystemConfigValue = any;
+export type SystemConfigValue = string | number | boolean;
 
 // dot notation matches path in `SystemConfig`
 export enum SystemConfigKey {
@@ -22,6 +23,18 @@ export enum SystemConfigKey {
   FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate',
   FFMPEG_TWO_PASS = 'ffmpeg.twoPass',
   FFMPEG_TRANSCODE = 'ffmpeg.transcode',
+
+  JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency',
+  JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency',
+  JOB_VIDEO_CONVERSION_CONCURRENCY = 'job.videoConversion.concurrency',
+  JOB_OBJECT_TAGGING_CONCURRENCY = 'job.objectTagging.concurrency',
+  JOB_RECOGNIZE_FACES_CONCURRENCY = 'job.recognizeFaces.concurrency',
+  JOB_CLIP_ENCODING_CONCURRENCY = 'job.clipEncoding.concurrency',
+  JOB_BACKGROUND_TASK_CONCURRENCY = 'job.backgroundTask.concurrency',
+  JOB_STORAGE_TEMPLATE_MIGRATION_CONCURRENCY = 'job.storageTemplateMigration.concurrency',
+  JOB_SEARCH_CONCURRENCY = 'job.search.concurrency',
+  JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency',
+
   OAUTH_ENABLED = 'oauth.enabled',
   OAUTH_ISSUER_URL = 'oauth.issuerUrl',
   OAUTH_CLIENT_ID = 'oauth.clientId',
@@ -32,7 +45,9 @@ export enum SystemConfigKey {
   OAUTH_AUTO_REGISTER = 'oauth.autoRegister',
   OAUTH_MOBILE_OVERRIDE_ENABLED = 'oauth.mobileOverrideEnabled',
   OAUTH_MOBILE_REDIRECT_URI = 'oauth.mobileRedirectUri',
+
   PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled',
+
   STORAGE_TEMPLATE = 'storageTemplate.template',
 }
 
@@ -55,6 +70,7 @@ export interface SystemConfig {
     twoPass: boolean;
     transcode: TranscodePreset;
   };
+  job: Record<QueueName, { concurrency: number }>;
   oauth: {
     enabled: boolean;
     issuerUrl: string;

server/libs/infra/src/infra.config.ts

@@ -1,5 +1,6 @@
 import { QueueName } from '@app/domain';
-import { BullModuleOptions } from '@nestjs/bull';
+import { RegisterQueueOptions } from '@nestjs/bullmq';
+import { QueueOptions } from 'bullmq';
 import { RedisOptions } from 'ioredis';
 import { InitOptions } from 'local-reverse-geocoder';
 import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
@@ -26,9 +27,9 @@ function parseRedisConfig(): RedisOptions {
 
 export const redisConfig: RedisOptions = parseRedisConfig();
 
-export const bullConfig: BullModuleOptions = {
+export const bullConfig: QueueOptions = {
   prefix: 'immich_bull',
-  redis: redisConfig,
+  connection: redisConfig,
   defaultJobOptions: {
     attempts: 3,
     removeOnComplete: true,
@@ -36,7 +37,7 @@ export const bullConfig: BullModuleOptions = {
   },
 };
 
-export const bullQueues: BullModuleOptions[] = Object.values(QueueName).map((name) => ({ name }));
+export const bullQueues: RegisterQueueOptions[] = Object.values(QueueName).map((name) => ({ name }));
 
 function parseTypeSenseConfig(): ConfigurationOptions {
   const typesenseURL = process.env.TYPESENSE_URL;

server/libs/infra/src/infra.module.ts

@@ -21,7 +21,7 @@ import {
   IUserRepository,
   IUserTokenRepository,
 } from '@app/domain';
-import { BullModule } from '@nestjs/bull';
+import { BullModule } from '@nestjs/bullmq';
 import { Global, Module, Provider } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { TypeOrmModule } from '@nestjs/typeorm';

server/libs/infra/src/repositories/job.repository.ts

@@ -1,13 +1,33 @@
 import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName, QueueStatus } from '@app/domain';
-import { getQueueToken } from '@nestjs/bull';
-import { Injectable } from '@nestjs/common';
+import { getQueueToken } from '@nestjs/bullmq';
+import { Injectable, Logger } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
-import { JobOptions, Queue, type JobCounts as BullJobCounts } from 'bull';
+import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq';
+import { bullConfig } from '../infra.config';
 
 @Injectable()
 export class JobRepository implements IJobRepository {
+  private workers: Partial<Record<QueueName, Worker>> = {};
+  private logger = new Logger(JobRepository.name);
+
   constructor(private moduleRef: ModuleRef) {}
 
+  addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise<void>) {
+    const workerHandler: Processor = async (job: Job) => handler(job as JobItem);
+    const workerOptions: WorkerOptions = { ...bullConfig, concurrency };
+    this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions);
+  }
+
+  setConcurrency(queueName: QueueName, concurrency: number) {
+    const worker = this.workers[queueName];
+    if (!worker) {
+      this.logger.warn(`Unable to set queue concurrency, worker not found: '${queueName}'`);
+      return;
+    }
+
+    worker.concurrency = concurrency;
+  }
+
   async getQueueStatus(name: QueueName): Promise<QueueStatus> {
     const queue = this.getQueue(name);
 
@@ -26,13 +46,18 @@ export class JobRepository implements IJobRepository {
   }
 
   empty(name: QueueName) {
-    return this.getQueue(name).empty();
+    return this.getQueue(name).drain();
   }
 
   getJobCounts(name: QueueName): Promise<JobCounts> {
-    // Typecast needed because the `paused` key is missing from Bull's
-    // type definition. Can be removed once fixed upstream.
-    return this.getQueue(name).getJobCounts() as Promise<BullJobCounts & { paused: number }>;
+    return this.getQueue(name).getJobCounts(
+      'active',
+      'completed',
+      'failed',
+      'delayed',
+      'waiting',
+      'paused',
+    ) as unknown as Promise<JobCounts>;
   }
 
   async queue(item: JobItem): Promise<void> {
@@ -43,7 +68,7 @@ export class JobRepository implements IJobRepository {
     await this.getQueue(JOBS_TO_QUEUE[jobName]).add(jobName, jobData, jobOptions);
   }
 
-  private getJobOptions(item: JobItem): JobOptions | null {
+  private getJobOptions(item: JobItem): JobsOptions | null {
     switch (item.name) {
       case JobName.GENERATE_FACE_THUMBNAIL:
         return { priority: 1 };

server/libs/infra/src/repositories/system-config.repository.ts

@@ -9,7 +9,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
     private repository: Repository<SystemConfigEntity>,
   ) {}
 
-  load(): Promise<SystemConfigEntity<string | boolean | number>[]> {
+  load(): Promise<SystemConfigEntity[]> {
     return this.repository.find();
   }