Added machine learning microservice and object detection (#76)

microservices/src/app.controller.ts

@@ -1,12 +0,0 @@
-import { Controller, Get } from '@nestjs/common';
-import { AppService } from './app.service';
-
-@Controller()
-export class AppController {
-  constructor(private readonly appService: AppService) {}
-
-  @Get()
-  getHello(): string {
-    return this.appService.getHello();
-  }
-}

microservices/src/app.module.ts

@@ -1,10 +1,16 @@
 import { Module } from '@nestjs/common';
-import { AppController } from './app.controller';
-import { AppService } from './app.service';
+import { ImageClassifierModule } from './image-classifier/image-classifier.module';
+import { databaseConfig } from './config/database.config';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ObjectDetectionModule } from './object-detection/object-detection.module';
 
 @Module({
-  imports: [],
-  controllers: [AppController],
-  providers: [AppService],
+  imports: [
+    TypeOrmModule.forRoot(databaseConfig),
+    ImageClassifierModule,
+    ObjectDetectionModule,
+  ],
+  controllers: [],
+  providers: [],
 })
 export class AppModule {}

microservices/src/app.service.ts

@@ -1,9 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-@Injectable()
-export class AppService {
-  getHello(): string {
-    console.log('Hello World 123');
-    return 'Hello World!';
-  }
-}

microservices/src/config/database.config.ts

@@ -0,0 +1,11 @@
+import { TypeOrmModuleOptions } from '@nestjs/typeorm';
+
+export const databaseConfig: TypeOrmModuleOptions = {
+  type: 'postgres',
+  host: 'immich_postgres',
+  port: 5432,
+  username: process.env.DB_USERNAME,
+  password: process.env.DB_PASSWORD,
+  database: process.env.DB_DATABASE_NAME,
+  synchronize: false,
+};

microservices/src/image-classifier/image-classifier.controller.ts

@@ -0,0 +1,14 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ImageClassifierService } from './image-classifier.service';
+
+@Controller('image-classifier')
+export class ImageClassifierController {
+  constructor(
+    private readonly imageClassifierService: ImageClassifierService,
+  ) {}
+
+  @Post('/tagImage')
+  async tagImage(@Body('thumbnailPath') thumbnailPath: string) {
+    return await this.imageClassifierService.tagImage(thumbnailPath);
+  }
+}

microservices/src/image-classifier/image-classifier.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ImageClassifierService } from './image-classifier.service';
+import { ImageClassifierController } from './image-classifier.controller';
+
+@Module({
+  controllers: [ImageClassifierController],
+  providers: [ImageClassifierService],
+})
+export class ImageClassifierModule {}

microservices/src/image-classifier/image-classifier.service.ts

@@ -0,0 +1,48 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as mobilenet from '@tensorflow-models/mobilenet';
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import * as tf from '@tensorflow/tfjs-node';
+import * as fs from 'fs';
+
+@Injectable()
+export class ImageClassifierService {
+  private readonly MOBILENET_VERSION = 2;
+  private readonly MOBILENET_ALPHA = 1.0;
+
+  private mobileNetModel: mobilenet.MobileNet;
+
+  constructor() {
+    Logger.log(
+      `Running Node TensorFlow Version : ${tf.version['tfjs']}`,
+      'ImageClassifier',
+    );
+    mobilenet
+      .load({
+        version: this.MOBILENET_VERSION,
+        alpha: this.MOBILENET_ALPHA,
+      })
+      .then((mobilenetModel) => (this.mobileNetModel = mobilenetModel));
+  }
+
+  async tagImage(thumbnailPath: string) {
+    try {
+      const isExist = fs.existsSync(thumbnailPath);
+      if (isExist) {
+        const tags = [];
+        const image = fs.readFileSync(thumbnailPath);
+        const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
+        const predictions = await this.mobileNetModel.classify(decodedImage);
+
+        for (const prediction of predictions) {
+          if (prediction.probability >= 0.1) {
+            tags.push(...prediction.className.split(',').map((e) => e.trim()));
+          }
+        }
+
+        return tags;
+      }
+    } catch (e) {
+      console.log('Error reading file ', e);
+    }
+  }
+}

microservices/src/main.ts

@@ -1,15 +1,10 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
-import { AppService } from './app.service';
 
 async function bootstrap() {
-  const app = await NestFactory.createApplicationContext(AppModule);
+  const app = await NestFactory.create(AppModule);
 
-  const appService = app.get(AppService);
-
-  appService.getHello();
-
-  await app.close();
+  await app.listen(3001);
 }
 
 bootstrap();

microservices/src/object-detection/object-detection.controller.ts

@@ -0,0 +1,14 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ObjectDetectionService } from './object-detection.service';
+
+@Controller('object-detection')
+export class ObjectDetectionController {
+  constructor(
+    private readonly objectDetectionService: ObjectDetectionService,
+  ) {}
+
+  @Post('/detectObject')
+  async detectObject(@Body('thumbnailPath') thumbnailPath: string) {
+    return await this.objectDetectionService.detectObject(thumbnailPath);
+  }
+}

microservices/src/object-detection/object-detection.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ObjectDetectionService } from './object-detection.service';
+import { ObjectDetectionController } from './object-detection.controller';
+
+@Module({
+  controllers: [ObjectDetectionController],
+  providers: [ObjectDetectionService],
+})
+export class ObjectDetectionModule {}

microservices/src/object-detection/object-detection.service.ts

@@ -0,0 +1,38 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import * as tf from '@tensorflow/tfjs-node';
+import * as fs from 'fs';
+
+@Injectable()
+export class ObjectDetectionService {
+  private cocoSsdModel: cocoSsd.ObjectDetection;
+
+  constructor() {
+    Logger.log(
+      `Running Node TensorFlow Version : ${tf.version['tfjs']}`,
+      'ObjectDetection',
+    );
+    cocoSsd.load().then((model) => (this.cocoSsdModel = model));
+  }
+  async detectObject(thumbnailPath: string) {
+    try {
+      const isExist = fs.existsSync(thumbnailPath);
+      if (isExist) {
+        const tags = new Set();
+        const image = fs.readFileSync(thumbnailPath);
+        const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
+        const predictions = await this.cocoSsdModel.detect(decodedImage);
+
+        for (const result of predictions) {
+          if (result.score > 0.5) {
+            tags.add(result.class);
+          }
+        }
+
+        return [...tags];
+      }
+    } catch (e) {
+      console.log('Error reading file ', e);
+    }
+  }
+}