Implemented image tagging using TensorFlow InceptionV3 (#28)

* Refactor docker-compose to its own folder
* Added FastAPI development environment
* Added support for GPU in docker file
* Added image classification
* creating endpoint for smart Image info
* added logo with white background on ios
* Added endpoint and trigger for image tagging
* Classify image and save into database
* Update readme

server/src/modules/background-task/background-task.module.ts

@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
+import { SmartInfoEntity } from '../../api-v1/asset/entities/smart-info.entity';
 import { BackgroundTaskProcessor } from './background-task.processor';
 import { BackgroundTaskService } from './background-task.service';
 
@@ -16,7 +17,7 @@ import { BackgroundTaskService } from './background-task.service';
         removeOnFail: false,
       },
     }),
-    TypeOrmModule.forFeature([AssetEntity, ExifEntity]),
+    TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]),
   ],
   providers: [BackgroundTaskService, BackgroundTaskProcessor],
   exports: [BackgroundTaskService],

server/src/modules/background-task/background-task.processor.ts

@@ -9,6 +9,8 @@ import { readFile } from 'fs/promises';
 import fs from 'fs';
 import { Logger } from '@nestjs/common';
 import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
+import axios from 'axios';
+import { SmartInfoEntity } from '../../api-v1/asset/entities/smart-info.entity';
 
 @Processor('background-task')
 export class BackgroundTaskProcessor {
@@ -16,6 +18,9 @@ export class BackgroundTaskProcessor {
     @InjectRepository(AssetEntity)
     private assetRepository: Repository<AssetEntity>,
 
+    @InjectRepository(SmartInfoEntity)
+    private smartInfoRepository: Repository<SmartInfoEntity>,
+
     @InjectRepository(ExifEntity)
     private exifRepository: Repository<ExifEntity>,
 
@@ -76,4 +81,18 @@ export class BackgroundTaskProcessor {
       });
     });
   }
+
+  @Process('tag-image')
+  async tagImage(job) {
+    const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
+    const res = await axios.post('http://immich_tf_fastapi:8000/tagImage', { thumbnail_path: thumbnailPath });
+
+    if (res.status == 200) {
+      const smartInfo = new SmartInfoEntity();
+      smartInfo.assetId = asset.id;
+      smartInfo.tags = [...res.data];
+
+      this.smartInfoRepository.save(smartInfo);
+    }
+  }
 }

server/src/modules/background-task/background-task.service.ts

@@ -32,4 +32,15 @@ export class BackgroundTaskService {
       { jobId: randomUUID() },
     );
   }
+
+  async tagImage(thumbnailPath: string, asset: AssetEntity) {
+    await this.backgroundTaskQueue.add(
+      'tag-image',
+      {
+        thumbnailPath,
+        asset,
+      },
+      { jobId: randomUUID() },
+    );
+  }
 }

server/src/modules/image-optimize/image-optimize.module.ts

@@ -1,20 +1,17 @@
 import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { join } from 'path';
-import { AssetModule } from '../../api-v1/asset/asset.module';
-import { AssetService } from '../../api-v1/asset/asset.service';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
-import { CommunicationGateway } from '../../api-v1/communication/communication.gateway';
 import { CommunicationModule } from '../../api-v1/communication/communication.module';
-import { UserEntity } from '../../api-v1/user/entities/user.entity';
-import { ImmichJwtModule } from '../immich-jwt/immich-jwt.module';
+import { BackgroundTaskModule } from '../background-task/background-task.module';
+import { BackgroundTaskService } from '../background-task/background-task.service';
 import { ImageOptimizeProcessor } from './image-optimize.processor';
 import { AssetOptimizeService } from './image-optimize.service';
 
 @Module({
   imports: [
     CommunicationModule,
+    BackgroundTaskModule,
     BullModule.registerQueue({
       name: 'optimize',
       defaultJobOptions: {
@@ -23,10 +20,17 @@ import { AssetOptimizeService } from './image-optimize.service';
         removeOnFail: false,
       },
     }),
-
+    BullModule.registerQueue({
+      name: 'background-task',
+      defaultJobOptions: {
+        attempts: 3,
+        removeOnComplete: true,
+        removeOnFail: false,
+      },
+    }),
     TypeOrmModule.forFeature([AssetEntity]),
   ],
-  providers: [AssetOptimizeService, ImageOptimizeProcessor],
+  providers: [AssetOptimizeService, ImageOptimizeProcessor, BackgroundTaskService],
   exports: [AssetOptimizeService],
 })
 export class ImageOptimizeModule {}

server/src/modules/image-optimize/image-optimize.processor.ts

@@ -11,6 +11,7 @@ import { APP_UPLOAD_LOCATION } from '../../constants/upload_location.constant';
 import { WebSocketServer } from '@nestjs/websockets';
 import { Socket, Server as SocketIoServer } from 'socket.io';
 import { CommunicationGateway } from '../../api-v1/communication/communication.gateway';
+import { BackgroundTaskService } from '../background-task/background-task.service';
 
 @Processor('optimize')
 export class ImageOptimizeProcessor {
@@ -18,6 +19,8 @@ export class ImageOptimizeProcessor {
     private wsCommunicateionGateway: CommunicationGateway,
     @InjectRepository(AssetEntity)
     private assetRepository: Repository<AssetEntity>,
+
+    private backgroundTaskService: BackgroundTaskService,
   ) {}
 
   @Process('resize-image')
@@ -58,11 +61,15 @@ export class ImageOptimizeProcessor {
             }
 
             const res = await this.assetRepository.update(savedAsset, { resizePath: desitnation });
+
             if (res.affected) {
               this.wsCommunicateionGateway.server
                 .to(savedAsset.userId)
                 .emit('on_upload_success', JSON.stringify(savedAsset));
             }
+
+            // Tag Image
+            this.backgroundTaskService.tagImage(desitnation, savedAsset);
           });
       } else {
         sharp(data)
@@ -79,6 +86,9 @@ export class ImageOptimizeProcessor {
                 .to(savedAsset.userId)
                 .emit('on_upload_success', JSON.stringify(savedAsset));
             }
+
+            // Tag Image
+            this.backgroundTaskService.tagImage(resizePath, savedAsset);
           });
       }
     });
@@ -107,12 +117,18 @@ export class ImageOptimizeProcessor {
         filename: `${filename}.png`,
       })
       .on('end', async (a) => {
+        const thumbnailPath = `${resizeDir}/${filename}.png`;
+
         const res = await this.assetRepository.update(savedAsset, { resizePath: `${resizeDir}/${filename}.png` });
+
         if (res.affected) {
           this.wsCommunicateionGateway.server
             .to(savedAsset.userId)
             .emit('on_upload_success', JSON.stringify(savedAsset));
         }
+
+        // Tag Image
+        this.backgroundTaskService.tagImage(thumbnailPath, savedAsset);
       });
 
     return 'ok';