Added machine learning microservice and object detection (#76)

server/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "immich",
-  "version": "0.0.1",
+  "version": "1.3.2",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "immich",
-      "version": "0.0.1",
+      "version": "1.3.2",
       "license": "UNLICENSED",
       "dependencies": {
         "@mapbox/mapbox-sdk": "^0.13.3",
@@ -1547,6 +1547,66 @@
         }
       }
     },
+    "node_modules/@nestjs/microservices": {
+      "version": "8.4.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-8.4.3.tgz",
+      "integrity": "sha512-/ZT5wo1s65J9Cqp2g5eNrYO34VH7/qUkDu4jJyZCT61I9UqpO49J3+1YIAHfmJJzHcrenjgt1sBtlFhwPR3Lgg==",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "iterare": "1.2.1",
+        "json-socket": "0.3.0",
+        "tslib": "2.3.1"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nest"
+      },
+      "peerDependencies": {
+        "@grpc/grpc-js": "*",
+        "@nestjs/common": "^8.0.0",
+        "@nestjs/core": "^8.0.0",
+        "@nestjs/websockets": "^8.0.0",
+        "amqp-connection-manager": "*",
+        "amqplib": "*",
+        "cache-manager": "*",
+        "kafkajs": "*",
+        "mqtt": "*",
+        "nats": "*",
+        "redis": "*",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@grpc/grpc-js": {
+          "optional": true
+        },
+        "@nestjs/websockets": {
+          "optional": true
+        },
+        "amqp-connection-manager": {
+          "optional": true
+        },
+        "amqplib": {
+          "optional": true
+        },
+        "cache-manager": {
+          "optional": true
+        },
+        "kafkajs": {
+          "optional": true
+        },
+        "mqtt": {
+          "optional": true
+        },
+        "nats": {
+          "optional": true
+        },
+        "redis": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@nestjs/passport": {
       "version": "8.1.0",
       "resolved": "https://registry.npmjs.org/@nestjs/passport/-/passport-8.1.0.tgz",
@@ -7053,6 +7113,13 @@
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
+    "node_modules/json-socket": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/json-socket/-/json-socket-0.3.0.tgz",
+      "integrity": "sha512-jc8ZbUnYIWdxERFWQKVgwSLkGSe+kyzvmYxwNaRgx/c8NNyuHes4UHnPM3LUrAFXUx1BhNJ94n1h/KCRlbvV0g==",
+      "optional": true,
+      "peer": true
+    },
     "node_modules/json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
@@ -11943,6 +12010,18 @@
       "integrity": "sha512-NFvofzSinp00j5rzUd4tf+xi9od6383iY0JP7o0Bnu1fuItAUkWBgc4EKuIQ3D+c2QI3i9pG1kDWAeY27EMGtg==",
       "requires": {}
     },
+    "@nestjs/microservices": {
+      "version": "8.4.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-8.4.3.tgz",
+      "integrity": "sha512-/ZT5wo1s65J9Cqp2g5eNrYO34VH7/qUkDu4jJyZCT61I9UqpO49J3+1YIAHfmJJzHcrenjgt1sBtlFhwPR3Lgg==",
+      "optional": true,
+      "peer": true,
+      "requires": {
+        "iterare": "1.2.1",
+        "json-socket": "0.3.0",
+        "tslib": "2.3.1"
+      }
+    },
     "@nestjs/passport": {
       "version": "8.1.0",
       "resolved": "https://registry.npmjs.org/@nestjs/passport/-/passport-8.1.0.tgz",
@@ -16243,6 +16322,13 @@
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
+    "json-socket": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/json-socket/-/json-socket-0.3.0.tgz",
+      "integrity": "sha512-jc8ZbUnYIWdxERFWQKVgwSLkGSe+kyzvmYxwNaRgx/c8NNyuHes4UHnPM3LUrAFXUx1BhNJ94n1h/KCRlbvV0g==",
+      "optional": true,
+      "peer": true
+    },
     "json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",

server/src/api-v1/asset/asset.controller.ts

@@ -16,13 +16,12 @@ import {
 } from '@nestjs/common';
 import { JwtAuthGuard } from '../../modules/immich-jwt/guards/jwt-auth.guard';
 import { AssetService } from './asset.service';
-import { FileFieldsInterceptor, FilesInterceptor } from '@nestjs/platform-express';
+import { FileFieldsInterceptor } from '@nestjs/platform-express';
 import { multerOption } from '../../config/multer-option.config';
 import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
 import { CreateAssetDto } from './dto/create-asset.dto';
 import { ServeFileDto } from './dto/serve-file.dto';
-import { AssetOptimizeService } from '../../modules/image-optimize/image-optimize.service';
-import { AssetEntity, AssetType } from './entities/asset.entity';
+import { AssetEntity } from './entities/asset.entity';
 import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
 import { Response as Res } from 'express';
 import { GetNewAssetQueryDto } from './dto/get-new-asset-query.dto';
@@ -61,6 +60,7 @@ export class AssetController {
       if (uploadFiles.thumbnailData != null) {
         await this.assetService.updateThumbnailInfo(savedAsset.id, uploadFiles.thumbnailData[0].path);
         await this.backgroundTaskService.tagImage(uploadFiles.thumbnailData[0].path, savedAsset);
+        await this.backgroundTaskService.detectObject(uploadFiles.thumbnailData[0].path, savedAsset);
       }
 
       await this.backgroundTaskService.extractExif(savedAsset, file.originalname, file.size);
@@ -81,6 +81,11 @@ export class AssetController {
     return this.assetService.serveFile(authUser, query, res, headers);
   }
 
+  @Get('/allObjects')
+  async getCuratedObject(@GetAuthUser() authUser: AuthUserDto) {
+    return this.assetService.getCuratedObject(authUser);
+  }
+
   @Get('/allLocation')
   async getCuratedLocation(@GetAuthUser() authUser: AuthUserDto) {
     return this.assetService.getCuratedLocation(authUser);

server/src/api-v1/asset/asset.service.ts

@@ -3,9 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { MoreThan, Repository } from 'typeorm';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { CreateAssetDto } from './dto/create-asset.dto';
-import { UpdateAssetDto } from './dto/update-asset.dto';
 import { AssetEntity, AssetType } from './entities/asset.entity';
-import _, { result } from 'lodash';
+import _ from 'lodash';
 import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
 import { GetAllAssetReponseDto } from './dto/get-all-asset-response.dto';
 import { createReadStream, stat } from 'fs';
@@ -44,9 +43,7 @@ export class AssetService {
     asset.duration = assetInfo.duration;
 
     try {
-      const res = await this.assetRepository.save(asset);
-
-      return res;
+      return await this.assetRepository.save(asset);
     } catch (e) {
       Logger.error(`Error Create New Asset ${e}`, 'createUserAsset');
     }
@@ -68,13 +65,11 @@ export class AssetService {
 
   public async getAllAssetsNoPagination(authUser: AuthUserDto) {
     try {
-      const assets = await this.assetRepository
+      return await this.assetRepository
         .createQueryBuilder('a')
         .where('a."userId" = :userId', { userId: authUser.id })
         .orderBy('a."createdAt"::date', 'DESC')
         .getMany();
-
-      return assets;
     } catch (e) {
       Logger.error(e, 'getAllAssets');
     }
@@ -226,10 +221,10 @@ export class AssetService {
   }
 
   public async deleteAssetById(authUser: AuthUserDto, assetIds: DeleteAssetDto) {
-    let result = [];
+    const result = [];
 
     const target = assetIds.ids;
-    for (let assetId of target) {
+    for (const assetId of target) {
       const res = await this.assetRepository.delete({
         id: assetId,
         userId: authUser.id,
@@ -251,11 +246,11 @@ export class AssetService {
     return result;
   }
 
-  async getAssetSearchTerm(authUser: AuthUserDto): Promise<String[]> {
-    const possibleSearchTerm = new Set<String>();
+  async getAssetSearchTerm(authUser: AuthUserDto): Promise<string[]> {
+    const possibleSearchTerm = new Set<string>();
     const rows = await this.assetRepository.query(
       `
-      select distinct si.tags, e.orientation, e."lensModel", e.make, e.model , a.type, e.city, e.state, e.country
+      select distinct si.tags, si.objects, e.orientation, e."lensModel", e.make, e.model , a.type, e.city, e.state, e.country
       from assets a
       left join exif e on a.id = e."assetId"
       left join smart_info si on a.id = si."assetId"
@@ -268,6 +263,9 @@ export class AssetService {
       // tags
       row['tags']?.map((tag) => possibleSearchTerm.add(tag?.toLowerCase()));
 
+      // objects
+      row['objects']?.map((object) => possibleSearchTerm.add(object?.toLowerCase()));
+
       // asset's tyoe
       possibleSearchTerm.add(row['type']?.toLowerCase());
 
@@ -300,18 +298,17 @@ export class AssetService {
     WHERE a."userId" = $1
        AND 
        (
-         TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR 
+         TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
+         TO_TSVECTOR('english', ARRAY_TO_STRING(si.objects, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
          e.exif_text_searchable_column @@ PLAINTO_TSQUERY('english', $2)
         );
     `;
 
-    const rows = await this.assetRepository.query(query, [authUser.id, searchAssetDto.searchTerm]);
-
-    return rows;
+    return await this.assetRepository.query(query, [authUser.id, searchAssetDto.searchTerm]);
   }
 
   async getCuratedLocation(authUser: AuthUserDto) {
-    const rows = await this.assetRepository.query(
+    return await this.assetRepository.query(
       `
         select distinct on (e.city) a.id, e.city, a."resizePath", a."deviceAssetId", a."deviceId"
         from assets a
@@ -322,7 +319,18 @@ export class AssetService {
       `,
       [authUser.id],
     );
+  }
 
-    return rows;
+  async getCuratedObject(authUser: AuthUserDto) {
+    return await this.assetRepository.query(
+      `
+        select distinct on (unnest(si.objects)) a.id, unnest(si.objects) as "object", a."resizePath", a."deviceAssetId", a."deviceId"
+        from assets a
+        left join smart_info si on a.id = si."assetId"
+        where a."userId" = $1 
+        and si.objects is not null
+      `,
+      [authUser.id],
+    );
   }
 }

server/src/api-v1/asset/entities/smart-info.entity.ts

@@ -13,6 +13,9 @@ export class SmartInfoEntity {
   @Column({ type: 'text', array: true, nullable: true })
   tags: string[];
 
+  @Column({ type: 'text', array: true, nullable: true })
+  objects: string[];
+
   @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
   @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
   asset: SmartInfoEntity;

server/src/app.module.ts

@@ -5,7 +5,6 @@ import { UserModule } from './api-v1/user/user.module';
 import { AssetModule } from './api-v1/asset/asset.module';
 import { AuthModule } from './api-v1/auth/auth.module';
 import { ImmichJwtModule } from './modules/immich-jwt/immich-jwt.module';
-import { JwtModule } from '@nestjs/jwt';
 import { DeviceInfoModule } from './api-v1/device-info/device-info.module';
 import { AppLoggerMiddleware } from './middlewares/app-logger.middleware';
 import { ConfigModule, ConfigService } from '@nestjs/config';
@@ -26,14 +25,12 @@ import { CommunicationModule } from './api-v1/communication/communication.module
     ImmichJwtModule,
     DeviceInfoModule,
     BullModule.forRootAsync({
-      imports: [ConfigModule],
-      useFactory: async (configService: ConfigService) => ({
+      useFactory: async () => ({
         redis: {
           host: 'immich_redis',
           port: 6379,
         },
       }),
-      inject: [ConfigService],
     }),
 
     ImageOptimizeModule,

server/src/migration/1648317474768-AddObjectColumnToSmartInfo.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddObjectColumnToSmartInfo1648317474768
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE smart_info
+        ADD COLUMN objects text[];
+
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE smart_info
+        DROP COLUMN objects;
+    `);
+  }
+}

server/src/modules/background-task/background-task.processor.ts

@@ -6,7 +6,7 @@ import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import { ConfigService } from '@nestjs/config';
 import exifr from 'exifr';
 import { readFile } from 'fs/promises';
-import fs, { rmSync } from 'fs';
+import fs from 'fs';
 import { Logger } from '@nestjs/common';
 import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
 import axios from 'axios';
@@ -114,14 +114,37 @@ export class BackgroundTaskProcessor {
   @Process('tag-image')
   async tagImage(job) {
     const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
-    const res = await axios.post('http://immich_tf_fastapi:8000/tagImage', { thumbnail_path: thumbnailPath });
 
-    if (res.status == 200) {
+    const res = await axios.post('http://immich_microservices:3001/image-classifier/tagImage', {
+      thumbnailPath: thumbnailPath,
+    });
+
+    if (res.status == 201 && res.data.length > 0) {
       const smartInfo = new SmartInfoEntity();
       smartInfo.assetId = asset.id;
       smartInfo.tags = [...res.data];
 
-      await this.smartInfoRepository.save(smartInfo);
+      await this.smartInfoRepository.upsert(smartInfo, {
+        conflictPaths: ['assetId'],
+      });
+    }
+  }
+
+  @Process('detect-object')
+  async detectObject(job) {
+    const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
+
+    const res = await axios.post('http://immich_microservices:3001/object-detection/detectObject', {
+      thumbnailPath: thumbnailPath,
+    });
+
+    if (res.status == 201 && res.data.length > 0) {
+      const smartInfo = new SmartInfoEntity();
+      smartInfo.assetId = asset.id;
+      smartInfo.objects = [...res.data];
+      await this.smartInfoRepository.upsert(smartInfo, {
+        conflictPaths: ['assetId'],
+      });
     }
   }
 }

server/src/modules/background-task/background-task.service.ts

@@ -43,4 +43,15 @@ export class BackgroundTaskService {
       { jobId: randomUUID() },
     );
   }
+
+  async detectObject(thumbnailPath: string, asset: AssetEntity) {
+    await this.backgroundTaskQueue.add(
+      'detect-object',
+      {
+        thumbnailPath,
+        asset,
+      },
+      { jobId: randomUUID() },
+    );
+  }
 }