feat(server)!: search via typesense (#1778)

* build: add typesense to docker

* feat(server): typesense search

* feat(web): search

* fix(web): show api error response message

* chore: search tests

* chore: regenerate open api

* fix: disable typesense on e2e

* fix: number properties for open api (dart)

* fix: e2e test

* fix: change lat/lng from floats to typesense geopoint

* dev: Add smartInfo relation to findAssetById to be able to query against it

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/apps/immich/src/api-v1/album/album.service.spec.ts

@@ -2,7 +2,7 @@ import { AlbumService } from './album.service';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { BadRequestException, NotFoundException, ForbiddenException } from '@nestjs/common';
 import { AlbumEntity, AssetEntity, UserEntity } from '@app/infra';
-import { AlbumResponseDto, ICryptoRepository, mapUser } from '@app/domain';
+import { AlbumResponseDto, ICryptoRepository, IJobRepository, JobName, mapUser } from '@app/domain';
 import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
 import { IAlbumRepository } from './album-repository';
 import { DownloadService } from '../../modules/download/download.service';
@@ -10,6 +10,7 @@ import { ISharedLinkRepository } from '@app/domain';
 import {
   assetEntityStub,
   newCryptoRepositoryMock,
+  newJobRepositoryMock,
   newSharedLinkRepositoryMock,
   userEntityStub,
 } from '@app/domain/../test';
@@ -20,6 +21,7 @@ describe('Album service', () => {
   let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
   let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
   let cryptoMock: jest.Mocked<ICryptoRepository>;
+  let jobMock: jest.Mocked<IJobRepository>;
 
   const authUser: AuthUserDto = Object.freeze({
     id: '1111',
@@ -139,12 +141,14 @@ describe('Album service', () => {
     };
 
     cryptoMock = newCryptoRepositoryMock();
+    jobMock = newJobRepositoryMock();
 
     sut = new AlbumService(
       albumRepositoryMock,
       sharedLinkRepositoryMock,
       downloadServiceMock as DownloadService,
       cryptoMock,
+      jobMock,
     );
   });
 
@@ -158,6 +162,7 @@ describe('Album service', () => {
 
     expect(result.id).toEqual(albumEntity.id);
     expect(result.albumName).toEqual(albumEntity.albumName);
+    expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.SEARCH_INDEX_ALBUM, data: { album: albumEntity } });
   });
 
   it('gets list of albums for auth user', async () => {
@@ -291,9 +296,8 @@ describe('Album service', () => {
     const updatedAlbumName = 'new album name';
     const updatedAlbumThumbnailAssetId = '69d2f917-0b31-48d8-9d7d-673b523f1aac';
     albumRepositoryMock.get.mockImplementation(() => Promise.resolve<AlbumEntity>(albumEntity));
-    albumRepositoryMock.updateAlbum.mockImplementation(() =>
-      Promise.resolve<AlbumEntity>({ ...albumEntity, albumName: updatedAlbumName }),
-    );
+    const updatedAlbum = { ...albumEntity, albumName: updatedAlbumName };
+    albumRepositoryMock.updateAlbum.mockResolvedValue(updatedAlbum);
 
     const result = await sut.updateAlbumInfo(
       authUser,
@@ -311,6 +315,7 @@ describe('Album service', () => {
       albumName: updatedAlbumName,
       albumThumbnailAssetId: updatedAlbumThumbnailAssetId,
     });
+    expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.SEARCH_INDEX_ALBUM, data: { album: updatedAlbum } });
   });
 
   it('prevents updating a not owned album (shared with auth user)', async () => {

server/apps/immich/src/api-v1/album/album.service.ts

@@ -6,7 +6,7 @@ import { AddUsersDto } from './dto/add-users.dto';
 import { RemoveAssetsDto } from './dto/remove-assets.dto';
 import { UpdateAlbumDto } from './dto/update-album.dto';
 import { GetAlbumsDto } from './dto/get-albums.dto';
-import { AlbumResponseDto, mapAlbum, mapAlbumExcludeAssetInfo } from '@app/domain';
+import { AlbumResponseDto, IJobRepository, JobName, mapAlbum, mapAlbumExcludeAssetInfo } from '@app/domain';
 import { IAlbumRepository } from './album-repository';
 import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
 import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
@@ -27,6 +27,7 @@ export class AlbumService {
     @Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
     private downloadService: DownloadService,
     @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
+    @Inject(IJobRepository) private jobRepository: IJobRepository,
   ) {
     this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository);
   }
@@ -56,6 +57,7 @@ export class AlbumService {
 
   async create(authUser: AuthUserDto, createAlbumDto: CreateAlbumDto): Promise<AlbumResponseDto> {
     const albumEntity = await this.albumRepository.create(authUser.id, createAlbumDto);
+    await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { album: albumEntity } });
     return mapAlbum(albumEntity);
   }
 
@@ -105,6 +107,7 @@ export class AlbumService {
     }
 
     await this.albumRepository.delete(album);
+    await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ALBUM, data: { id: albumId } });
   }
 
   async removeUserFromAlbum(authUser: AuthUserDto, albumId: string, userId: string | 'me'): Promise<void> {
@@ -171,6 +174,9 @@ export class AlbumService {
     }
 
     const updatedAlbum = await this.albumRepository.updateAlbum(album, updateAlbumDto);
+
+    await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { album: updatedAlbum } });
+
     return mapAlbum(updatedAlbum);
   }
 

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -252,7 +252,7 @@ export class AssetRepository implements IAssetRepository {
       where: {
         id: assetId,
       },
-      relations: ['exifInfo', 'tags', 'sharedLinks'],
+      relations: ['exifInfo', 'tags', 'sharedLinks', 'smartInfo'],
     });
   }
 

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -445,6 +445,8 @@ describe('AssetService', () => {
       ]);
 
       expect(jobMock.queue.mock.calls).toEqual([
+        [{ name: JobName.SEARCH_REMOVE_ASSET, data: { id: 'asset1' } }],
+        [{ name: JobName.SEARCH_REMOVE_ASSET, data: { id: 'asset2' } }],
         [
           {
             name: JobName.DELETE_FILES,

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -170,6 +170,8 @@ export class AssetService {
 
     const updatedAsset = await this._assetRepository.update(authUser.id, asset, dto);
 
+    await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { asset: updatedAsset } });
+
     return mapAsset(updatedAsset);
   }
 
@@ -425,6 +427,7 @@ export class AssetService {
 
       try {
         await this._assetRepository.remove(asset);
+        await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { id } });
 
         result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });
         deleteQueue.push(asset.originalPath, asset.webpPath, asset.resizePath);

server/apps/immich/src/app.module.ts

@@ -1,5 +1,5 @@
 import { immichAppConfig } from '@app/common/config';
-import { Module } from '@nestjs/common';
+import { Module, OnModuleInit } from '@nestjs/common';
 import { AssetModule } from './api-v1/asset/asset.module';
 import { ConfigModule } from '@nestjs/config';
 import { ServerInfoModule } from './api-v1/server-info/server-info.module';
@@ -9,13 +9,14 @@ import { ScheduleModule } from '@nestjs/schedule';
 import { ScheduleTasksModule } from './modules/schedule-tasks/schedule-tasks.module';
 import { JobModule } from './api-v1/job/job.module';
 import { TagModule } from './api-v1/tag/tag.module';
-import { DomainModule } from '@app/domain';
+import { DomainModule, SearchService } from '@app/domain';
 import { InfraModule } from '@app/infra';
 import {
   APIKeyController,
   AuthController,
   DeviceInfoController,
   OAuthController,
+  SearchController,
   ShareController,
   SystemConfigController,
   UserController,
@@ -46,16 +47,21 @@ import { AuthGuard } from './middlewares/auth.guard';
     TagModule,
   ],
   controllers: [
-    //
     AppController,
     APIKeyController,
     AuthController,
     DeviceInfoController,
     OAuthController,
+    SearchController,
     ShareController,
     SystemConfigController,
     UserController,
   ],
   providers: [{ provide: APP_GUARD, useExisting: AuthGuard }, AuthGuard],
 })
-export class AppModule {}
+export class AppModule implements OnModuleInit {
+  constructor(private searchService: SearchService) {}
+  async onModuleInit() {
+    await this.searchService.bootstrap();
+  }
+}

server/apps/immich/src/controllers/index.ts

@@ -2,6 +2,7 @@ export * from './api-key.controller';
 export * from './auth.controller';
 export * from './device-info.controller';
 export * from './oauth.controller';
+export * from './search.controller';
 export * from './share.controller';
 export * from './system-config.controller';
 export * from './user.controller';

server/apps/immich/src/controllers/search.controller.ts

@@ -0,0 +1,27 @@
+import { AuthUserDto, SearchConfigResponseDto, SearchDto, SearchResponseDto, SearchService } from '@app/domain';
+import { Controller, Get, Query, ValidationPipe } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { GetAuthUser } from '../decorators/auth-user.decorator';
+import { Authenticated } from '../decorators/authenticated.decorator';
+
+@ApiTags('Search')
+@Authenticated()
+@Controller('search')
+export class SearchController {
+  constructor(private readonly searchService: SearchService) {}
+
+  @Authenticated()
+  @Get()
+  async search(
+    @GetAuthUser() authUser: AuthUserDto,
+    @Query(new ValidationPipe({ transform: true })) dto: SearchDto,
+  ): Promise<SearchResponseDto> {
+    return this.searchService.search(authUser, dto);
+  }
+
+  @Authenticated()
+  @Get('config')
+  getSearchConfig(): SearchConfigResponseDto {
+    return this.searchService.getConfig();
+  }
+}

server/apps/immich/src/main.ts

@@ -11,7 +11,7 @@ import { RedisIoAdapter } from './middlewares/redis-io.adapter.middleware';
 import { json } from 'body-parser';
 import { patchOpenAPI } from './utils/patch-open-api.util';
 import { getLogLevels, MACHINE_LEARNING_ENABLED } from '@app/common';
-import { IMMICH_ACCESS_COOKIE } from '@app/domain';
+import { IMMICH_ACCESS_COOKIE, SearchService } from '@app/domain';
 
 const logger = new Logger('ImmichServer');
 
@@ -73,6 +73,9 @@ async function bootstrap() {
     );
   });
 
+  const searchService = app.get(SearchService);
+
   logger.warn(`Machine learning is ${MACHINE_LEARNING_ENABLED ? 'enabled' : 'disabled'}`);
+  logger.warn(`Search is ${searchService.isEnabled() ? 'enabled' : 'disabled'}`);
 }
 bootstrap();

server/apps/microservices/src/microservices.module.ts

@@ -7,6 +7,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import {
   BackgroundTaskProcessor,
   MachineLearningProcessor,
+  SearchIndexProcessor,
   StorageTemplateMigrationProcessor,
   ThumbnailGeneratorProcessor,
 } from './processors';
@@ -26,6 +27,7 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
     MachineLearningProcessor,
     StorageTemplateMigrationProcessor,
     BackgroundTaskProcessor,
+    SearchIndexProcessor,
   ],
 })
 export class MicroservicesModule {}

server/apps/microservices/src/processors.ts

@@ -1,12 +1,15 @@
 import {
   AssetService,
+  IAlbumJob,
   IAssetJob,
   IAssetUploadedJob,
   IDeleteFilesJob,
+  IDeleteJob,
   IUserDeletionJob,
   JobName,
   MediaService,
   QueueName,
+  SearchService,
   SmartInfoService,
   StorageService,
   StorageTemplateService,
@@ -61,6 +64,41 @@ export class MachineLearningProcessor {
   }
 }
 
+@Processor(QueueName.SEARCH)
+export class SearchIndexProcessor {
+  constructor(private searchService: SearchService) {}
+
+  @Process(JobName.SEARCH_INDEX_ALBUMS)
+  async onIndexAlbums() {
+    await this.searchService.handleIndexAlbums();
+  }
+
+  @Process(JobName.SEARCH_INDEX_ASSETS)
+  async onIndexAssets() {
+    await this.searchService.handleIndexAssets();
+  }
+
+  @Process(JobName.SEARCH_INDEX_ALBUM)
+  async onIndexAlbum(job: Job<IAlbumJob>) {
+    await this.searchService.handleIndexAlbum(job.data);
+  }
+
+  @Process(JobName.SEARCH_INDEX_ASSET)
+  async onIndexAsset(job: Job<IAssetJob>) {
+    await this.searchService.handleIndexAsset(job.data);
+  }
+
+  @Process(JobName.SEARCH_REMOVE_ALBUM)
+  async onRemoveAlbum(job: Job<IDeleteJob>) {
+    await this.searchService.handleRemoveAlbum(job.data);
+  }
+
+  @Process(JobName.SEARCH_REMOVE_ASSET)
+  async onRemoveAsset(job: Job<IDeleteJob>) {
+    await this.searchService.handleRemoveAsset(job.data);
+  }
+}
+
 @Processor(QueueName.STORAGE_TEMPLATE_MIGRATION)
 export class StorageTemplateMigrationProcessor {
   constructor(private storageTemplateService: StorageTemplateService) {}

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -1,18 +1,26 @@
+import {
+  AssetCore,
+  IAssetRepository,
+  IAssetUploadedJob,
+  IReverseGeocodingJob,
+  ISearchRepository,
+  JobName,
+  QueueName,
+} from '@app/domain';
 import { AssetEntity, AssetType, ExifEntity } from '@app/infra';
-import { IReverseGeocodingJob, IAssetUploadedJob, QueueName, JobName, IAssetRepository } from '@app/domain';
 import { Process, Processor } from '@nestjs/bull';
 import { Inject, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Job } from 'bull';
+import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored';
 import ffmpeg from 'fluent-ffmpeg';
+import { getName } from 'i18n-iso-countries';
+import geocoder, { InitOptions } from 'local-reverse-geocoder';
+import fs from 'node:fs';
 import path from 'path';
 import sharp from 'sharp';
 import { Repository } from 'typeorm/repository/Repository';
-import geocoder, { InitOptions } from 'local-reverse-geocoder';
-import { getName } from 'i18n-iso-countries';
-import fs from 'node:fs';
-import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored';
 
 interface ImmichTags extends Tags {
   ContentIdentifier?: string;
@@ -71,13 +79,19 @@ export type GeoData = {
 export class MetadataExtractionProcessor {
   private logger = new Logger(MetadataExtractionProcessor.name);
   private isGeocodeInitialized = false;
+  private assetCore: AssetCore;
+
   constructor(
-    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(IAssetRepository) assetRepository: IAssetRepository,
+    @Inject(ISearchRepository) searchRepository: ISearchRepository,
+
     @InjectRepository(ExifEntity)
     private exifRepository: Repository<ExifEntity>,
 
     configService: ConfigService,
   ) {
+    this.assetCore = new AssetCore(assetRepository, searchRepository);
+
     if (!configService.get('DISABLE_REVERSE_GEOCODING')) {
       this.logger.log('Initializing Reverse Geocoding');
       geocoderInit({
@@ -175,20 +189,11 @@ export class MetadataExtractionProcessor {
       newExif.longitude = exifData?.GPSLongitude || null;
       newExif.livePhotoCID = exifData?.MediaGroupUUID || null;
 
-      await this.assetRepository.save({
-        id: asset.id,
-        fileCreatedAt: fileCreatedAt?.toISOString(),
-      });
-
       if (newExif.livePhotoCID && !asset.livePhotoVideoId) {
-        const motionAsset = await this.assetRepository.findLivePhotoMatch(
-          newExif.livePhotoCID,
-          asset.id,
-          AssetType.VIDEO,
-        );
+        const motionAsset = await this.assetCore.findLivePhotoMatch(newExif.livePhotoCID, asset.id, AssetType.VIDEO);
         if (motionAsset) {
-          await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id });
-          await this.assetRepository.save({ id: motionAsset.id, isVisible: false });
+          await this.assetCore.save({ id: asset.id, livePhotoVideoId: motionAsset.id });
+          await this.assetCore.save({ id: motionAsset.id, isVisible: false });
         }
       }
 
@@ -226,6 +231,7 @@ export class MetadataExtractionProcessor {
       }
 
       await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
+      await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() });
     } catch (error: any) {
       this.logger.error(`Error extracting EXIF ${error}`, error?.stack);
     }
@@ -292,14 +298,10 @@ export class MetadataExtractionProcessor {
       newExif.livePhotoCID = exifData?.ContentIdentifier || null;
 
       if (newExif.livePhotoCID) {
-        const photoAsset = await this.assetRepository.findLivePhotoMatch(
-          newExif.livePhotoCID,
-          asset.id,
-          AssetType.IMAGE,
-        );
+        const photoAsset = await this.assetCore.findLivePhotoMatch(newExif.livePhotoCID, asset.id, AssetType.IMAGE);
         if (photoAsset) {
-          await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
-          await this.assetRepository.save({ id: asset.id, isVisible: false });
+          await this.assetCore.save({ id: photoAsset.id, livePhotoVideoId: asset.id });
+          await this.assetCore.save({ id: asset.id, isVisible: false });
         }
       }
 
@@ -355,7 +357,7 @@ export class MetadataExtractionProcessor {
       }
 
       await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
-      await this.assetRepository.save({ id: asset.id, duration: durationString, fileCreatedAt });
+      await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt });
     } catch (err) {
       ``;
       // do nothing