feat: facial recognition (#2180)

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -210,7 +210,15 @@ export class AssetRepository implements IAssetRepository {
       where: {
         id: assetId,
       },
-      relations: ['exifInfo', 'tags', 'sharedLinks', 'smartInfo'],
+      relations: {
+        exifInfo: true,
+        tags: true,
+        sharedLinks: true,
+        smartInfo: true,
+        faces: {
+          person: true,
+        },
+      },
     });
   }
 
@@ -239,7 +247,14 @@ export class AssetRepository implements IAssetRepository {
   }
 
   get(id: string): Promise<AssetEntity | null> {
-    return this.assetRepository.findOne({ where: { id } });
+    return this.assetRepository.findOne({
+      where: { id },
+      relations: {
+        faces: {
+          person: true,
+        },
+      },
+    });
   }
 
   async create(
@@ -264,11 +279,6 @@ export class AssetRepository implements IAssetRepository {
     asset.isFavorite = dto.isFavorite ?? asset.isFavorite;
     asset.isArchived = dto.isArchived ?? asset.isArchived;
 
-    if (dto.tagIds) {
-      const tags = await this._tagRepository.getByIds(userId, dto.tagIds);
-      asset.tags = tags;
-    }
-
     if (asset.exifInfo != null) {
       asset.exifInfo.description = dto.description || '';
       await this.exifRepository.save(asset.exifInfo);
@@ -280,7 +290,12 @@ export class AssetRepository implements IAssetRepository {
       asset.exifInfo = exifInfo;
     }
 
-    return await this.assetRepository.save(asset);
+    await this.assetRepository.update(asset.id, {
+      isFavorite: asset.isFavorite,
+      isArchived: asset.isArchived,
+    });
+
+    return asset;
   }
 
   /**

server/apps/immich/src/api-v1/asset/asset.core.ts

@@ -38,6 +38,7 @@ export class AssetCore {
       tags: [],
       sharedLinks: [],
       originalFileName: parse(file.originalName).name,
+      faces: [],
     });
 
     await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -355,6 +355,14 @@ export class AssetService {
       }
 
       try {
+        if (asset.faces) {
+          await Promise.all(
+            asset.faces.map(({ assetId, personId }) =>
+              this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
+            ),
+          );
+        }
+
         await this._assetRepository.remove(asset);
         await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } });
 

server/apps/immich/src/app.cron-jobs.ts

@@ -1,13 +1,13 @@
-import { UserService } from '@app/domain';
+import { JobService } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
 @Injectable()
 export class AppCronJobs {
-  constructor(private userService: UserService) {}
+  constructor(private jobService: JobService) {}
 
-  @Cron(CronExpression.EVERY_DAY_AT_11PM)
-  async onQueueUserDeleteCheck() {
-    await this.userService.handleQueueUserDelete();
+  @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
+  async onNightlyJob() {
+    await this.jobService.handleNightlyJobs();
   }
 }

server/apps/immich/src/app.module.ts

@@ -10,6 +10,7 @@ import {
   AlbumController,
   APIKeyController,
   AuthController,
+  PersonController,
   JobController,
   OAuthController,
   PartnerController,
@@ -44,6 +45,7 @@ import { AppCronJobs } from './app.cron-jobs';
     SharedLinkController,
     SystemConfigController,
     UserController,
+    PersonController,
   ],
   providers: [
     //

server/apps/immich/src/controllers/index.ts

@@ -4,6 +4,7 @@ export * from './auth.controller';
 export * from './job.controller';
 export * from './oauth.controller';
 export * from './partner.controller';
+export * from './person.controller';
 export * from './search.controller';
 export * from './server-info.controller';
 export * from './shared-link.controller';

server/apps/immich/src/controllers/person.controller.ts

@@ -0,0 +1,57 @@
+import {
+  AssetResponseDto,
+  AuthUserDto,
+  ImmichReadStream,
+  PersonResponseDto,
+  PersonService,
+  PersonUpdateDto,
+} from '@app/domain';
+import { Body, Controller, Get, Param, Put, StreamableFile } from '@nestjs/common';
+import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import { GetAuthUser } from '../decorators/auth-user.decorator';
+
+import { Authenticated } from '../decorators/authenticated.decorator';
+import { UseValidation } from '../decorators/use-validation.decorator';
+import { UUIDParamDto } from './dto/uuid-param.dto';
+
+function asStreamableFile({ stream, type, length }: ImmichReadStream) {
+  return new StreamableFile(stream, { type, length });
+}
+
+@ApiTags('Person')
+@Controller('person')
+@Authenticated()
+@UseValidation()
+export class PersonController {
+  constructor(private service: PersonService) {}
+
+  @Get()
+  getAllPeople(@GetAuthUser() authUser: AuthUserDto): Promise<PersonResponseDto[]> {
+    return this.service.getAll(authUser);
+  }
+
+  @Get(':id')
+  getPerson(@GetAuthUser() authUser: AuthUserDto, @Param() { id }: UUIDParamDto): Promise<PersonResponseDto> {
+    return this.service.getById(authUser, id);
+  }
+
+  @Put(':id')
+  updatePerson(
+    @GetAuthUser() authUser: AuthUserDto,
+    @Param() { id }: UUIDParamDto,
+    @Body() dto: PersonUpdateDto,
+  ): Promise<PersonResponseDto> {
+    return this.service.update(authUser, id, dto);
+  }
+
+  @Get(':id/thumbnail')
+  @ApiOkResponse({ content: { 'application/octet-stream': { schema: { type: 'string', format: 'binary' } } } })
+  getPersonThumbnail(@GetAuthUser() authUser: AuthUserDto, @Param() { id }: UUIDParamDto) {
+    return this.service.getThumbnail(authUser, id).then(asStreamableFile);
+  }
+
+  @Get(':id/assets')
+  getPersonAssets(@GetAuthUser() authUser: AuthUserDto, @Param() { id }: UUIDParamDto): Promise<AssetResponseDto[]> {
+    return this.service.getAssets(authUser, id);
+  }
+}

server/apps/microservices/src/microservices.module.ts

@@ -6,6 +6,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import {
   BackgroundTaskProcessor,
   ClipEncodingProcessor,
+  FacialRecognitionProcessor,
   ObjectTaggingProcessor,
   SearchIndexProcessor,
   StorageTemplateMigrationProcessor,
@@ -29,6 +30,7 @@ import { MetadataExtractionProcessor } from './processors/metadata-extraction.pr
     StorageTemplateMigrationProcessor,
     BackgroundTaskProcessor,
     SearchIndexProcessor,
+    FacialRecognitionProcessor,
   ],
 })
 export class MicroservicesModule {}

server/apps/microservices/src/processors.ts

@@ -1,13 +1,17 @@
 import {
   AssetService,
+  FacialRecognitionService,
+  IAssetFaceJob,
   IAssetJob,
   IAssetUploadedJob,
   IBaseJob,
   IBulkEntityJob,
   IDeleteFilesJob,
+  IFaceThumbnailJob,
   IUserDeletionJob,
   JobName,
   MediaService,
+  PersonService,
   QueueName,
   SearchService,
   SmartInfoService,
@@ -23,6 +27,7 @@ import { Job } from 'bull';
 export class BackgroundTaskProcessor {
   constructor(
     private assetService: AssetService,
+    private personService: PersonService,
     private storageService: StorageService,
     private systemConfigService: SystemConfigService,
     private userService: UserService,
@@ -43,10 +48,20 @@ export class BackgroundTaskProcessor {
     await this.systemConfigService.refreshConfig();
   }
 
+  @Process(JobName.USER_DELETE_CHECK)
+  async onUserDeleteCheck() {
+    await this.userService.handleUserDeleteCheck();
+  }
+
   @Process(JobName.USER_DELETION)
   async onUserDelete(job: Job<IUserDeletionJob>) {
     await this.userService.handleUserDelete(job.data);
   }
+
+  @Process(JobName.PERSON_CLEANUP)
+  async onPersonCleanup() {
+    await this.personService.handlePersonCleanup();
+  }
 }
 
 @Processor(QueueName.OBJECT_TAGGING)
@@ -69,6 +84,26 @@ export class ObjectTaggingProcessor {
   }
 }
 
+@Processor(QueueName.RECOGNIZE_FACES)
+export class FacialRecognitionProcessor {
+  constructor(private facialRecognitionService: FacialRecognitionService) {}
+
+  @Process({ name: JobName.QUEUE_RECOGNIZE_FACES, concurrency: 1 })
+  async onQueueRecognizeFaces(job: Job<IBaseJob>) {
+    await this.facialRecognitionService.handleQueueRecognizeFaces(job.data);
+  }
+
+  @Process({ name: JobName.RECOGNIZE_FACES, concurrency: 1 })
+  async onRecognizeFaces(job: Job<IAssetJob>) {
+    await this.facialRecognitionService.handleRecognizeFaces(job.data);
+  }
+
+  @Process({ name: JobName.GENERATE_FACE_THUMBNAIL, concurrency: 1 })
+  async onGenerateFaceThumbnail(job: Job<IFaceThumbnailJob>) {
+    await this.facialRecognitionService.handleGenerateFaceThumbnail(job.data);
+  }
+}
+
 @Processor(QueueName.CLIP_ENCODING)
 export class ClipEncodingProcessor {
   constructor(private smartInfoService: SmartInfoService) {}
@@ -98,6 +133,11 @@ export class SearchIndexProcessor {
     await this.searchService.handleIndexAssets();
   }
 
+  @Process(JobName.SEARCH_INDEX_FACES)
+  async onIndexFaces() {
+    await this.searchService.handleIndexFaces();
+  }
+
   @Process(JobName.SEARCH_INDEX_ALBUM)
   onIndexAlbum(job: Job<IBulkEntityJob>) {
     this.searchService.handleIndexAlbum(job.data);
@@ -108,6 +148,11 @@ export class SearchIndexProcessor {
     this.searchService.handleIndexAsset(job.data);
   }
 
+  @Process(JobName.SEARCH_INDEX_FACE)
+  async onIndexFace(job: Job<IAssetFaceJob>) {
+    await this.searchService.handleIndexFace(job.data);
+  }
+
   @Process(JobName.SEARCH_REMOVE_ALBUM)
   onRemoveAlbum(job: Job<IBulkEntityJob>) {
     this.searchService.handleRemoveAlbum(job.data);
@@ -117,6 +162,11 @@ export class SearchIndexProcessor {
   onRemoveAsset(job: Job<IBulkEntityJob>) {
     this.searchService.handleRemoveAsset(job.data);
   }
+
+  @Process(JobName.SEARCH_REMOVE_FACE)
+  onRemoveFace(job: Job<IAssetFaceJob>) {
+    this.searchService.handleRemoveFace(job.data);
+  }
 }
 
 @Processor(QueueName.STORAGE_TEMPLATE_MIGRATION)