feat: facial recognition (#2180)

server/libs/infra/src/repositories/asset.repository.ts

@@ -15,6 +15,9 @@ export class AssetRepository implements IAssetRepository {
         exifInfo: true,
         smartInfo: true,
         tags: true,
+        faces: {
+          person: true,
+        },
       },
     });
   }
@@ -35,6 +38,9 @@ export class AssetRepository implements IAssetRepository {
         exifInfo: true,
         smartInfo: true,
         tags: true,
+        faces: {
+          person: true,
+        },
       },
     });
   }
@@ -48,6 +54,7 @@ export class AssetRepository implements IAssetRepository {
         owner: true,
         smartInfo: true,
         tags: true,
+        faces: true,
       },
     });
   }
@@ -129,6 +136,20 @@ export class AssetRepository implements IAssetRepository {
         };
         break;
 
+      case WithoutProperty.FACES:
+        relations = {
+          faces: true,
+        };
+        where = {
+          resizePath: IsNull(),
+          isVisible: true,
+          faces: {
+            assetId: IsNull(),
+            personId: IsNull(),
+          },
+        };
+        break;
+
       default:
         throw new Error(`Invalid getWithout property: ${property}`);
     }

server/libs/infra/src/repositories/face.repository.ts

@@ -0,0 +1,22 @@
+import { AssetFaceId, IFaceRepository } from '@app/domain';
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+import { AssetFaceEntity } from '../entities/asset-face.entity';
+
+@Injectable()
+export class FaceRepository implements IFaceRepository {
+  constructor(@InjectRepository(AssetFaceEntity) private repository: Repository<AssetFaceEntity>) {}
+
+  getAll(): Promise<AssetFaceEntity[]> {
+    return this.repository.find({ relations: { asset: true } });
+  }
+
+  getByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]> {
+    return this.repository.find({ where: ids, relations: { asset: true } });
+  }
+
+  create(entity: Partial<AssetFaceEntity>): Promise<AssetFaceEntity> {
+    return this.repository.save(entity);
+  }
+}

server/libs/infra/src/repositories/index.ts

@@ -3,12 +3,14 @@ export * from './api-key.repository';
 export * from './asset.repository';
 export * from './communication.repository';
 export * from './crypto.repository';
+export * from './face.repository';
 export * from './filesystem.provider';
 export * from './geocoding.repository';
 export * from './job.repository';
 export * from './machine-learning.repository';
 export * from './media.repository';
 export * from './partner.repository';
+export * from './person.repository';
 export * from './shared-link.repository';
 export * from './smart-info.repository';
 export * from './system-config.repository';

server/libs/infra/src/repositories/job.repository.ts

@@ -20,6 +20,7 @@ export class JobRepository implements IJobRepository {
     [QueueName.THUMBNAIL_GENERATION]: this.generateThumbnail,
     [QueueName.METADATA_EXTRACTION]: this.metadataExtraction,
     [QueueName.OBJECT_TAGGING]: this.objectTagging,
+    [QueueName.RECOGNIZE_FACES]: this.recognizeFaces,
     [QueueName.CLIP_ENCODING]: this.clipEmbedding,
     [QueueName.VIDEO_CONVERSION]: this.videoTranscode,
     [QueueName.BACKGROUND_TASK]: this.backgroundTask,
@@ -31,6 +32,7 @@ export class JobRepository implements IJobRepository {
     @InjectQueue(QueueName.OBJECT_TAGGING) private objectTagging: Queue<IAssetJob | IBaseJob>,
     @InjectQueue(QueueName.CLIP_ENCODING) private clipEmbedding: Queue<IAssetJob | IBaseJob>,
     @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IAssetUploadedJob | IBaseJob>,
+    @InjectQueue(QueueName.RECOGNIZE_FACES) private recognizeFaces: Queue<IAssetJob | IBaseJob>,
     @InjectQueue(QueueName.STORAGE_TEMPLATE_MIGRATION) private storageTemplateMigration: Queue,
     @InjectQueue(QueueName.THUMBNAIL_GENERATION) private generateThumbnail: Queue,
     @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IAssetJob | IBaseJob>,
@@ -91,6 +93,19 @@ export class JobRepository implements IJobRepository {
         await this.metadataExtraction.add(item.name, item.data);
         break;
 
+      case JobName.QUEUE_RECOGNIZE_FACES:
+      case JobName.RECOGNIZE_FACES:
+        await this.recognizeFaces.add(item.name, item.data);
+        break;
+
+      case JobName.GENERATE_FACE_THUMBNAIL:
+        await this.recognizeFaces.add(item.name, item.data, { priority: 1 });
+        break;
+
+      case JobName.PERSON_CLEANUP:
+        await this.backgroundTask.add(item.name);
+        break;
+
       case JobName.QUEUE_GENERATE_THUMBNAILS:
       case JobName.GENERATE_JPEG_THUMBNAIL:
       case JobName.GENERATE_WEBP_THUMBNAIL:
@@ -120,13 +135,19 @@ export class JobRepository implements IJobRepository {
 
       case JobName.SEARCH_INDEX_ASSETS:
       case JobName.SEARCH_INDEX_ALBUMS:
+      case JobName.SEARCH_INDEX_FACES:
         await this.searchIndex.add(item.name, {});
         break;
 
       case JobName.SEARCH_INDEX_ASSET:
       case JobName.SEARCH_INDEX_ALBUM:
+      case JobName.SEARCH_INDEX_FACE:
+        await this.searchIndex.add(item.name, item.data);
+        break;
+
       case JobName.SEARCH_REMOVE_ALBUM:
       case JobName.SEARCH_REMOVE_ASSET:
+      case JobName.SEARCH_REMOVE_FACE:
         await this.searchIndex.add(item.name, item.data);
         break;
 

server/libs/infra/src/repositories/machine-learning.repository.ts

@@ -1,4 +1,4 @@
-import { IMachineLearningRepository, MachineLearningInput, MACHINE_LEARNING_URL } from '@app/domain';
+import { DetectFaceResult, IMachineLearningRepository, MachineLearningInput, MACHINE_LEARNING_URL } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import axios from 'axios';
 
@@ -10,6 +10,10 @@ export class MachineLearningRepository implements IMachineLearningRepository {
     return client.post<string[]>('/image-classifier/tag-image', input).then((res) => res.data);
   }
 
+  detectFaces(input: MachineLearningInput): Promise<DetectFaceResult[]> {
+    return client.post<DetectFaceResult[]>('/facial-recognition/detect-faces', input).then((res) => res.data);
+  }
+
   detectObjects(input: MachineLearningInput): Promise<string[]> {
     return client.post<string[]>('/object-detection/detect-object', input).then((res) => res.data);
   }

server/libs/infra/src/repositories/media.repository.ts

@@ -1,4 +1,4 @@
-import { IMediaRepository, ResizeOptions, VideoInfo } from '@app/domain';
+import { CropOptions, IMediaRepository, ResizeOptions, VideoInfo } from '@app/domain';
 import { exiftool } from 'exiftool-vendored';
 import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
 import sharp from 'sharp';
@@ -7,11 +7,22 @@ import { promisify } from 'util';
 const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
 
 export class MediaRepository implements IMediaRepository {
+  crop(input: string, options: CropOptions): Promise<Buffer> {
+    return sharp(input, { failOnError: false })
+      .extract({
+        left: options.left,
+        top: options.top,
+        width: options.width,
+        height: options.height,
+      })
+      .toBuffer();
+  }
+
   extractThumbnailFromExif(input: string, output: string): Promise<void> {
     return exiftool.extractThumbnail(input, output);
   }
 
-  async resize(input: string, output: string, options: ResizeOptions): Promise<void> {
+  async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void> {
     switch (options.format) {
       case 'webp':
         await sharp(input, { failOnError: false })

server/libs/infra/src/repositories/person.repository.ts

@@ -0,0 +1,78 @@
+import { IPersonRepository, PersonSearchOptions } from '@app/domain';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
+
+export class PersonRepository implements IPersonRepository {
+  constructor(
+    @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
+    @InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
+    @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
+  ) {}
+
+  delete(entity: PersonEntity): Promise<PersonEntity | null> {
+    return this.personRepository.remove(entity);
+  }
+
+  async deleteAll(): Promise<number> {
+    const people = await this.personRepository.find();
+    await this.personRepository.remove(people);
+    return people.length;
+  }
+
+  getAll(userId: string, options?: PersonSearchOptions): Promise<PersonEntity[]> {
+    return this.personRepository
+      .createQueryBuilder('person')
+      .leftJoin('person.faces', 'face')
+      .where('person.ownerId = :userId', { userId })
+      .orderBy('COUNT(face.assetId)', 'DESC')
+      .having('COUNT(face.assetId) >= :faces', { faces: options?.minimumFaceCount || 1 })
+      .groupBy('person.id')
+      .getMany();
+  }
+
+  getAllWithoutFaces(): Promise<PersonEntity[]> {
+    return this.personRepository
+      .createQueryBuilder('person')
+      .leftJoin('person.faces', 'face')
+      .having('COUNT(face.assetId) = 0')
+      .groupBy('person.id')
+      .getMany();
+  }
+
+  getById(ownerId: string, personId: string): Promise<PersonEntity | null> {
+    return this.personRepository.findOne({ where: { id: personId, ownerId } });
+  }
+
+  getAssets(ownerId: string, personId: string): Promise<AssetEntity[]> {
+    return this.assetRepository.find({
+      where: {
+        ownerId,
+        faces: {
+          personId,
+        },
+        isVisible: true,
+      },
+      relations: {
+        faces: {
+          person: true,
+        },
+        exifInfo: true,
+      },
+      order: {
+        createdAt: 'ASC',
+      },
+      // TODO: remove after either (1) pagination or (2) time bucket is implemented for this query
+      take: 3000,
+    });
+  }
+
+  create(entity: Partial<PersonEntity>): Promise<PersonEntity> {
+    return this.personRepository.save(entity);
+  }
+
+  async update(entity: Partial<PersonEntity>): Promise<PersonEntity> {
+    const { id } = await this.personRepository.save(entity);
+    return this.personRepository.findOneByOrFail({ id });
+  }
+}

server/libs/infra/src/repositories/typesense.repository.ts

@@ -1,8 +1,10 @@
 import {
   ISearchRepository,
+  OwnedFaceEntity,
   SearchCollection,
   SearchCollectionIndexStatus,
   SearchExploreItem,
+  SearchFaceFilter,
   SearchFilter,
   SearchResult,
 } from '@app/domain';
@@ -12,9 +14,9 @@ import { catchError, filter, firstValueFrom, from, map, mergeMap, of, toArray }
 import { Client } from 'typesense';
 import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections';
 import { DocumentSchema, SearchResponse } from 'typesense/lib/Typesense/Documents';
-import { AlbumEntity, AssetEntity } from '../entities';
+import { AlbumEntity, AssetEntity, AssetFaceEntity } from '../entities';
 import { typesenseConfig } from '../infra.config';
-import { albumSchema, assetSchema } from '../typesense-schemas';
+import { albumSchema, assetSchema, faceSchema } from '../typesense-schemas';
 
 function removeNil<T extends Dictionary<any>>(item: T): T {
   _.forOwn(item, (value, key) => {
@@ -26,14 +28,21 @@ function removeNil<T extends Dictionary<any>>(item: T): T {
   return item;
 }
 
+interface MultiSearchError {
+  code: number;
+  error: string;
+}
+
 interface CustomAssetEntity extends AssetEntity {
   geo?: [number, number];
   motion?: boolean;
+  people?: string[];
 }
 
 const schemaMap: Record<SearchCollection, CollectionCreateSchema> = {
   [SearchCollection.ASSETS]: assetSchema,
   [SearchCollection.ALBUMS]: albumSchema,
+  [SearchCollection.FACES]: faceSchema,
 };
 
 const schemas = Object.entries(schemaMap) as [SearchCollection, CollectionCreateSchema][];
@@ -61,7 +70,7 @@ export class TypesenseRepository implements ISearchRepository {
   async setup(): Promise<void> {
     const collections = await this.client.collections().retrieve();
     for (const collection of collections) {
-      this.logger.debug(`${collection.name} => ${collection.num_documents}`);
+      this.logger.debug(`${collection.name} collection has ${collection.num_documents} documents`);
       // await this.client.collections(collection.name).delete();
     }
 
@@ -84,6 +93,7 @@ export class TypesenseRepository implements ISearchRepository {
     const migrationMap: SearchCollectionIndexStatus = {
       [SearchCollection.ASSETS]: false,
       [SearchCollection.ALBUMS]: false,
+      [SearchCollection.FACES]: false,
     };
 
     // check if alias is using the current schema
@@ -110,9 +120,13 @@ export class TypesenseRepository implements ISearchRepository {
     await this.import(SearchCollection.ASSETS, items, done);
   }
 
+  async importFaces(items: OwnedFaceEntity[], done: boolean): Promise<void> {
+    await this.import(SearchCollection.FACES, items, done);
+  }
+
   private async import(
     collection: SearchCollection,
-    items: AlbumEntity[] | AssetEntity[],
+    items: AlbumEntity[] | AssetEntity[] | OwnedFaceEntity[],
     done: boolean,
   ): Promise<void> {
     try {
@@ -198,6 +212,15 @@ export class TypesenseRepository implements ISearchRepository {
     await this.delete(SearchCollection.ASSETS, ids);
   }
 
+  async deleteFaces(ids: string[]): Promise<void> {
+    await this.delete(SearchCollection.FACES, ids);
+  }
+
+  async deleteAllFaces(): Promise<number> {
+    const records = await this.client.collections(faceSchema.name).documents().delete({ filter_by: 'ownerId:!=null' });
+    return records.num_deleted;
+  }
+
   async delete(collection: SearchCollection, ids: string[]): Promise<void> {
     await this.client
       .collections(schemaMap[collection].name)
@@ -232,6 +255,7 @@ export class TypesenseRepository implements ISearchRepository {
           'exifInfo.description',
           'smartInfo.tags',
           'smartInfo.objects',
+          'people',
         ].join(','),
         per_page: 250,
         facet_by: this.getFacetFieldNames(SearchCollection.ASSETS),
@@ -242,6 +266,22 @@ export class TypesenseRepository implements ISearchRepository {
     return this.asResponse(results, filters.debug);
   }
 
+  async searchFaces(input: number[], filters: SearchFaceFilter): Promise<SearchResult<AssetFaceEntity>> {
+    const { results } = await this.client.multiSearch.perform({
+      searches: [
+        {
+          collection: faceSchema.name,
+          q: '*',
+          vector_query: `embedding:([${input.join(',')}], k:5)`,
+          per_page: 250,
+          filter_by: this.buildFilterBy('ownerId', filters.ownerId, true),
+        } as any,
+      ],
+    });
+
+    return this.asResponse(results[0] as SearchResponse<AssetFaceEntity>);
+  }
+
   async vectorSearch(input: number[], filters: SearchFilter): Promise<SearchResult<AssetEntity>> {
     const { results } = await this.client.multiSearch.perform({
       searches: [
@@ -259,12 +299,23 @@ export class TypesenseRepository implements ISearchRepository {
     return this.asResponse(results[0] as SearchResponse<AssetEntity>, filters.debug);
   }
 
-  private asResponse<T extends DocumentSchema>(results: SearchResponse<T>, debug?: boolean): SearchResult<T> {
+  private asResponse<T extends DocumentSchema>(
+    resultsOrError: SearchResponse<T> | MultiSearchError,
+    debug?: boolean,
+  ): SearchResult<T> {
+    const { error, code } = resultsOrError as MultiSearchError;
+    if (error) {
+      throw new Error(`Typesense multi-search error: ${code} - ${error}`);
+    }
+
+    const results = resultsOrError as SearchResponse<T>;
+
     return {
       page: results.page,
       total: results.found,
       count: results.out_of,
       items: (results.hits || []).map((hit) => hit.document),
+      distances: (results.hits || []).map((hit: any) => hit.vector_distance),
       facets: (results.facet_counts || []).map((facet) => ({
         counts: facet.counts.map((item) => ({ count: item.count, value: item.value })),
         fieldName: facet.field_name as string,
@@ -306,12 +357,17 @@ export class TypesenseRepository implements ISearchRepository {
     }
   }
 
-  private patch(collection: SearchCollection, items: AssetEntity[] | AlbumEntity[]) {
-    return items.map((item) =>
-      collection === SearchCollection.ASSETS
-        ? this.patchAsset(item as AssetEntity)
-        : this.patchAlbum(item as AlbumEntity),
-    );
+  private patch(collection: SearchCollection, items: AssetEntity[] | AlbumEntity[] | OwnedFaceEntity[]) {
+    return items.map((item) => {
+      switch (collection) {
+        case SearchCollection.ASSETS:
+          return this.patchAsset(item as AssetEntity);
+        case SearchCollection.ALBUMS:
+          return this.patchAlbum(item as AlbumEntity);
+        case SearchCollection.FACES:
+          return this.patchFace(item as OwnedFaceEntity);
+      }
+    });
   }
 
   private patchAlbum(album: AlbumEntity): AlbumEntity {
@@ -327,9 +383,17 @@ export class TypesenseRepository implements ISearchRepository {
       custom = { ...custom, geo: [lat, lng] };
     }
 
+    const people = asset.faces?.map((face) => face.person.name).filter((name) => name) || [];
+    if (people.length) {
+      custom = { ...custom, people };
+    }
     return removeNil({ ...custom, motion: !!asset.livePhotoVideoId });
   }
 
+  private patchFace(face: OwnedFaceEntity): OwnedFaceEntity {
+    return removeNil(face);
+  }
+
   private getFacetFieldNames(collection: SearchCollection) {
     return (schemaMap[collection].fields || [])
       .filter((field) => field.facet)