WIP refactor container and queuing system (#206)

* refactor microservices to machine-learning

* Update tGithub issue template with correct task syntax

* Added microservices container

* Communicate between service based on queue system

* added dependency

* Fixed problem with having to import BullQueue into the individual service

* Added todo

* refactor server into monorepo with microservices

* refactor database and entity to library

* added simple migration

* Move migrations and database config to library

* Migration works in library

* Cosmetic change in logging message

* added user dto

* Fixed issue with testing not able to find the shared library

* Clean up library mapping path

* Added webp generator to microservices

* Update Github Action build latest

* Fixed issue NPM cannot install due to conflict witl Bull Queue

* format project with prettier

* Modified docker-compose file

* Add GH Action for Staging build:

* Fixed GH action job name

* Modified GH Action to only build & push latest when pushing to main

* Added Test 2e2 Github Action

* Added Test 2e2 Github Action

* Implemented microservice to extract exif

* Added cronjob to scan and generate webp thumbnail  at midnight

* Refactor to ireduce hit time to database when running microservices

* Added error handling to asset services that handle read file from disk

* Added video transcoding queue to process one video at a time

* Fixed loading spinner on web while loading covering the info panel

* Add mechanism to show new release announcement to web and mobile app (#209)

* Added changelog page

* Fixed issues based on PR comments

* Fixed issue with video transcoding run on the server

* Change entry point content for backward combatibility when starting up server

* Added announcement box

* Added error handling to failed silently when the app version checking is not able to make the request to GITHUB

* Added new version announcement overlay

* Update message

* Added messages

* Added logic to check and show announcement

* Add method to handle saving new version

* Added button to dimiss the acknowledge message

* Up version for deployment to the app store

machine-learning/.dockerignore

@@ -0,0 +1,4 @@
+node_modules/
+upload/
+dist/
+

machine-learning/.eslintrc.js

@@ -0,0 +1,24 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+  },
+};

machine-learning/.gitignore

@@ -0,0 +1,37 @@
+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+pnpm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+upload/

machine-learning/.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}

machine-learning/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:16-bullseye-slim 
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /usr/src/app
+
+COPY package.json package-lock.json ./
+
+RUN apt-get update
+RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
+
+RUN npm install
+
+COPY . .
+
+RUN npm run build

machine-learning/README.md

@@ -0,0 +1,4 @@
+
+# Microservices for Immich
+
+## Image Classifier

machine-learning/entrypoint.sh

@@ -0,0 +1,2 @@
+# npm run typeorm migration:run
+npm run build && npm run start:prod

machine-learning/nest-cli.json

@@ -0,0 +1,4 @@
+{
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src"
+}

machine-learning/package.json

@@ -0,0 +1,83 @@
+{
+  "name": "nest_microservices",
+  "version": "0.0.1",
+  "description": "",
+  "author": "",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "prebuild": "rimraf dist",
+    "build": "nest build",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "start": "nest start",
+    "start:dev": "nest start --watch",
+    "start:debug": "nest start --debug --watch",
+    "start:prod": "node dist/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
+    "test:e2e": "jest --config ./test/jest-e2e.json"
+  },
+  "dependencies": {
+    "@nestjs/common": "^8.0.0",
+    "@nestjs/core": "^8.0.0",
+    "@nestjs/mapped-types": "^1.0.1",
+    "@nestjs/platform-express": "^8.0.0",
+    "@nestjs/typeorm": "^8.0.3",
+    "@tensorflow-models/coco-ssd": "^2.2.2",
+    "@tensorflow-models/mobilenet": "^2.1.0",
+    "@tensorflow/tfjs": "^3.15.0",
+    "@tensorflow/tfjs-converter": "^3.15.0",
+    "@tensorflow/tfjs-core": "^3.15.0",
+    "@tensorflow/tfjs-node": "^3.15.0",
+    "@tensorflow/tfjs-node-gpu": "^3.15.0",
+    "@trpc/server": "^9.20.3",
+    "pg": "^8.7.3",
+    "reflect-metadata": "^0.1.13",
+    "rimraf": "^3.0.2",
+    "rxjs": "^7.2.0",
+    "typeorm": "^0.2.45"
+  },
+  "devDependencies": {
+    "@nestjs/cli": "^8.2.4",
+    "@nestjs/schematics": "^8.0.0",
+    "@nestjs/testing": "^8.0.0",
+    "@types/express": "^4.17.13",
+    "@types/jest": "27.4.1",
+    "@types/node": "^16.0.0",
+    "@types/supertest": "^2.0.11",
+    "@typescript-eslint/eslint-plugin": "^5.0.0",
+    "@typescript-eslint/parser": "^5.0.0",
+    "eslint": "^8.0.1",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "jest": "^27.2.5",
+    "prettier": "^2.3.2",
+    "source-map-support": "^0.5.20",
+    "supertest": "^6.1.3",
+    "ts-jest": "^27.0.3",
+    "ts-loader": "^9.2.3",
+    "ts-node": "^10.0.0",
+    "tsconfig-paths": "^3.10.1",
+    "typescript": "^4.3.5"
+  },
+  "jest": {
+    "moduleFileExtensions": [
+      "js",
+      "json",
+      "ts"
+    ],
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": [
+      "**/*.(t|j)s"
+    ],
+    "coverageDirectory": "../coverage",
+    "testEnvironment": "node"
+  }
+}

machine-learning/src/app.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { ImageClassifierModule } from './image-classifier/image-classifier.module';
+import { databaseConfig } from './config/database.config';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ObjectDetectionModule } from './object-detection/object-detection.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forRoot(databaseConfig),
+    ImageClassifierModule,
+    ObjectDetectionModule,
+  ],
+  controllers: [],
+  providers: [],
+})
+export class AppModule {}

machine-learning/src/config/database.config.ts

@@ -0,0 +1,11 @@
+import { TypeOrmModuleOptions } from '@nestjs/typeorm';
+
+export const databaseConfig: TypeOrmModuleOptions = {
+  type: 'postgres',
+  host: process.env.DB_HOSTNAME || 'immich_postgres',
+  port: 5432,
+  username: process.env.DB_USERNAME,
+  password: process.env.DB_PASSWORD,
+  database: process.env.DB_DATABASE_NAME,
+  synchronize: false,
+};

machine-learning/src/image-classifier/image-classifier.controller.ts

@@ -0,0 +1,14 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ImageClassifierService } from './image-classifier.service';
+
+@Controller('image-classifier')
+export class ImageClassifierController {
+  constructor(
+    private readonly imageClassifierService: ImageClassifierService,
+  ) { }
+
+  @Post('/tag-image')
+  async tagImage(@Body('thumbnailPath') thumbnailPath: string) {
+    return await this.imageClassifierService.tagImage(thumbnailPath);
+  }
+}

machine-learning/src/image-classifier/image-classifier.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ImageClassifierService } from './image-classifier.service';
+import { ImageClassifierController } from './image-classifier.controller';
+
+@Module({
+  controllers: [ImageClassifierController],
+  providers: [ImageClassifierService],
+})
+export class ImageClassifierModule {}

machine-learning/src/image-classifier/image-classifier.service.ts

@@ -0,0 +1,49 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as mobilenet from '@tensorflow-models/mobilenet';
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import * as tf from '@tensorflow/tfjs-node';
+import * as fs from 'fs';
+
+@Injectable()
+export class ImageClassifierService {
+  private readonly MOBILENET_VERSION = 2;
+  private readonly MOBILENET_ALPHA = 1.0;
+
+  private mobileNetModel: mobilenet.MobileNet;
+
+  constructor() {
+    Logger.log(
+      `Running Node TensorFlow Version : ${tf.version['tfjs']}`,
+      'ImageClassifier',
+    );
+    mobilenet
+      .load({
+        version: this.MOBILENET_VERSION,
+        alpha: this.MOBILENET_ALPHA,
+      })
+      .then((mobilenetModel) => (this.mobileNetModel = mobilenetModel));
+  }
+
+  async tagImage(thumbnailPath: string) {
+    try {
+      const isExist = fs.existsSync(thumbnailPath);
+      if (isExist) {
+        const tags = [];
+        const image = fs.readFileSync(thumbnailPath);
+        const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
+        const predictions = await this.mobileNetModel.classify(decodedImage);
+
+        for (const prediction of predictions) {
+          if (prediction.probability >= 0.1) {
+            tags.push(...prediction.className.split(',').map((e) => e.trim()));
+          }
+        }
+
+        tf.dispose(decodedImage);
+        return tags;
+      }
+    } catch (e) {
+      console.log('Error reading file ', e);
+    }
+  }
+}

machine-learning/src/main.ts

@@ -0,0 +1,25 @@
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from './app.module';
+import { Logger } from '@nestjs/common';
+
+async function bootstrap() {
+  const app = await NestFactory.create(AppModule);
+
+  await app.listen(3001, () => {
+    if (process.env.NODE_ENV == 'development') {
+      Logger.log(
+        'Running Immich Machine Learning in DEVELOPMENT environment',
+        'IMMICH MICROSERVICES',
+      );
+    }
+
+    if (process.env.NODE_ENV == 'production') {
+      Logger.log(
+        'Running Immich Machine Learning in PRODUCTION environment',
+        'IMMICH MICROSERVICES',
+      );
+    }
+  });
+}
+
+bootstrap();

machine-learning/src/object-detection/object-detection.controller.ts

@@ -0,0 +1,15 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ObjectDetectionService } from './object-detection.service';
+import { Logger } from '@nestjs/common';
+
+@Controller('object-detection')
+export class ObjectDetectionController {
+  constructor(
+    private readonly objectDetectionService: ObjectDetectionService,
+  ) { }
+
+  @Post('/detect-object')
+  async detectObject(@Body('thumbnailPath') thumbnailPath: string) {
+    return await this.objectDetectionService.detectObject(thumbnailPath);
+  }
+}

machine-learning/src/object-detection/object-detection.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ObjectDetectionService } from './object-detection.service';
+import { ObjectDetectionController } from './object-detection.controller';
+
+@Module({
+  controllers: [ObjectDetectionController],
+  providers: [ObjectDetectionService],
+})
+export class ObjectDetectionModule {}

machine-learning/src/object-detection/object-detection.service.ts

@@ -0,0 +1,39 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import * as tf from '@tensorflow/tfjs-node';
+import * as fs from 'fs';
+
+@Injectable()
+export class ObjectDetectionService {
+  private cocoSsdModel: cocoSsd.ObjectDetection;
+
+  constructor() {
+    Logger.log(
+      `Running Node TensorFlow Version : ${tf.version['tfjs']}`,
+      'ObjectDetection',
+    );
+    cocoSsd.load().then((model) => (this.cocoSsdModel = model));
+  }
+  async detectObject(thumbnailPath: string) {
+    try {
+      const isExist = fs.existsSync(thumbnailPath);
+      if (isExist) {
+        const tags = new Set();
+        const image = fs.readFileSync(thumbnailPath);
+        const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
+        const predictions = await this.cocoSsdModel.detect(decodedImage);
+
+        for (const result of predictions) {
+          if (result.score > 0.5) {
+            tags.add(result.class);
+          }
+        }
+
+        tf.dispose(decodedImage);
+        return [...tags];
+      }
+    } catch (e) {
+      console.log('Error reading file ', e);
+    }
+  }
+}

machine-learning/tsconfig.build.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+}

machine-learning/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "declaration": true,
+    "removeComments": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "es2017",
+    "sourceMap": true,
+    "outDir": "./dist",
+    "baseUrl": "./",
+    "incremental": true,
+    "skipLibCheck": true,
+    "strictNullChecks": false,
+    "noImplicitAny": false,
+    "strictBindCallApply": false,
+    "forceConsistentCasingInFileNames": false,
+    "noFallthroughCasesInSwitch": false
+  }
+}