WIP refactor container and queuing system (#206)

* refactor microservices to machine-learning

* Update tGithub issue template with correct task syntax

* Added microservices container

* Communicate between service based on queue system

* added dependency

* Fixed problem with having to import BullQueue into the individual service

* Added todo

* refactor server into monorepo with microservices

* refactor database and entity to library

* added simple migration

* Move migrations and database config to library

* Migration works in library

* Cosmetic change in logging message

* added user dto

* Fixed issue with testing not able to find the shared library

* Clean up library mapping path

* Added webp generator to microservices

* Update Github Action build latest

* Fixed issue NPM cannot install due to conflict witl Bull Queue

* format project with prettier

* Modified docker-compose file

* Add GH Action for Staging build:

* Fixed GH action job name

* Modified GH Action to only build & push latest when pushing to main

* Added Test 2e2 Github Action

* Added Test 2e2 Github Action

* Implemented microservice to extract exif

* Added cronjob to scan and generate webp thumbnail  at midnight

* Refactor to ireduce hit time to database when running microservices

* Added error handling to asset services that handle read file from disk

* Added video transcoding queue to process one video at a time

* Fixed loading spinner on web while loading covering the info panel

* Add mechanism to show new release announcement to web and mobile app (#209)

* Added changelog page

* Fixed issues based on PR comments

* Fixed issue with video transcoding run on the server

* Change entry point content for backward combatibility when starting up server

* Added announcement box

* Added error handling to failed silently when the app version checking is not able to make the request to GITHUB

* Added new version announcement overlay

* Update message

* Added messages

* Added logic to check and show announcement

* Add method to handle saving new version

* Added button to dimiss the acknowledge message

* Up version for deployment to the app store

server/libs/database/src/entities/asset-shared-album.entity.ts

@@ -0,0 +1,30 @@
+import { Column, Entity, JoinColumn, ManyToOne, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
+import { AssetEntity } from './asset.entity';
+import { SharedAlbumEntity } from './shared-album.entity';
+
+@Entity('asset_shared_album')
+@Unique('PK_unique_asset_in_album', ['albumId', 'assetId'])
+export class AssetSharedAlbumEntity {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @Column()
+  albumId: string;
+
+  @Column()
+  assetId: string;
+
+  @ManyToOne(() => SharedAlbumEntity, (sharedAlbum) => sharedAlbum.sharedAssets, {
+    onDelete: 'CASCADE',
+    nullable: true,
+  })
+  @JoinColumn({ name: 'albumId' })
+  albumInfo: SharedAlbumEntity;
+
+  @ManyToOne(() => AssetEntity, {
+    onDelete: 'CASCADE',
+    nullable: true,
+  })
+  @JoinColumn({ name: 'assetId' })
+  assetInfo: AssetEntity;
+}

server/libs/database/src/entities/asset.entity.ts

@@ -0,0 +1,62 @@
+import { Column, Entity, JoinColumn, OneToOne, PrimaryColumn, PrimaryGeneratedColumn, Unique } from 'typeorm';
+import { ExifEntity } from './exif.entity';
+import { SmartInfoEntity } from './smart-info.entity';
+
+@Entity('assets')
+@Unique(['deviceAssetId', 'userId', 'deviceId'])
+export class AssetEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  deviceAssetId: string;
+
+  @Column()
+  userId: string;
+
+  @Column()
+  deviceId: string;
+
+  @Column()
+  type: AssetType;
+
+  @Column()
+  originalPath: string;
+
+  @Column({ nullable: true })
+  resizePath: string;
+
+  @Column({ nullable: true })
+  webpPath: string;
+
+  @Column({ nullable: true })
+  encodedVideoPath: string;
+
+  @Column()
+  createdAt: string;
+
+  @Column()
+  modifiedAt: string;
+
+  @Column({ type: 'boolean', default: false })
+  isFavorite: boolean;
+
+  @Column({ nullable: true })
+  mimeType: string;
+
+  @Column({ nullable: true })
+  duration: string;
+
+  @OneToOne(() => ExifEntity, (exifEntity) => exifEntity.asset)
+  exifInfo: ExifEntity;
+
+  @OneToOne(() => SmartInfoEntity, (smartInfoEntity) => smartInfoEntity.asset)
+  smartInfo: SmartInfoEntity;
+}
+
+export enum AssetType {
+  IMAGE = 'IMAGE',
+  VIDEO = 'VIDEO',
+  AUDIO = 'AUDIO',
+  OTHER = 'OTHER',
+}

server/libs/database/src/entities/device-info.entity.ts

@@ -0,0 +1,32 @@
+import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, Unique } from 'typeorm';
+
+@Entity('device_info')
+@Unique(['userId', 'deviceId'])
+export class DeviceInfoEntity {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column()
+  userId: string;
+
+  @Column()
+  deviceId: string;
+
+  @Column()
+  deviceType: DeviceType;
+
+  @Column({ nullable: true })
+  notificationToken: string;
+
+  @CreateDateColumn()
+  createdAt: string;
+
+  @Column({ type: 'bool', default: false })
+  isAutoBackup: boolean;
+}
+
+export enum DeviceType {
+  IOS = 'IOS',
+  ANDROID = 'ANDROID',
+  WEB = 'WEB',
+}

server/libs/database/src/entities/exif.entity.ts

@@ -0,0 +1,76 @@
+import { Index, JoinColumn, OneToOne } from 'typeorm';
+import { Column } from 'typeorm/decorator/columns/Column';
+import { PrimaryGeneratedColumn } from 'typeorm/decorator/columns/PrimaryGeneratedColumn';
+import { Entity } from 'typeorm/decorator/entity/Entity';
+import { AssetEntity } from './asset.entity';
+
+@Entity('exif')
+export class ExifEntity {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @Index({ unique: true })
+  @Column({ type: 'uuid' })
+  assetId: string;
+
+  @Column({ nullable: true })
+  make: string;
+
+  @Column({ nullable: true })
+  model: string;
+
+  @Column({ nullable: true })
+  imageName: string;
+
+  @Column({ nullable: true })
+  exifImageWidth: number;
+
+  @Column({ nullable: true })
+  exifImageHeight: number;
+
+  @Column({ nullable: true })
+  fileSizeInByte: number;
+
+  @Column({ nullable: true })
+  orientation: string;
+
+  @Column({ type: 'timestamptz', nullable: true })
+  dateTimeOriginal: Date;
+
+  @Column({ type: 'timestamptz', nullable: true })
+  modifyDate: Date;
+
+  @Column({ nullable: true })
+  lensModel: string;
+
+  @Column({ type: 'float8', nullable: true })
+  fNumber: number;
+
+  @Column({ type: 'float8', nullable: true })
+  focalLength: number;
+
+  @Column({ nullable: true })
+  iso: number;
+
+  @Column({ type: 'float', nullable: true })
+  exposureTime: number;
+
+  @Column({ type: 'float', nullable: true })
+  latitude: number;
+
+  @Column({ type: 'float', nullable: true })
+  longitude: number;
+
+  @Column({ nullable: true })
+  city: string;
+
+  @Column({ nullable: true })
+  state: string;
+
+  @Column({ nullable: true })
+  country: string;
+
+  @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
+  @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
+  asset: ExifEntity;
+}

server/libs/database/src/entities/shared-album.entity.ts

@@ -0,0 +1,27 @@
+import { Column, CreateDateColumn, Entity, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
+import { AssetSharedAlbumEntity } from './asset-shared-album.entity';
+import { UserSharedAlbumEntity } from './user-shared-album.entity';
+
+@Entity('shared_albums')
+export class SharedAlbumEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  ownerId: string;
+
+  @Column({ default: 'Untitled Album' })
+  albumName: string;
+
+  @CreateDateColumn({ type: 'timestamptz' })
+  createdAt: string;
+
+  @Column({ comment: 'Asset ID to be used as thumbnail', nullable: true })
+  albumThumbnailAssetId: string;
+
+  @OneToMany(() => UserSharedAlbumEntity, (userSharedAlbums) => userSharedAlbums.albumInfo)
+  sharedUsers: UserSharedAlbumEntity[];
+
+  @OneToMany(() => AssetSharedAlbumEntity, (assetSharedAlbumEntity) => assetSharedAlbumEntity.albumInfo)
+  sharedAssets: AssetSharedAlbumEntity[];
+}

server/libs/database/src/entities/smart-info.entity.ts

@@ -0,0 +1,22 @@
+import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryGeneratedColumn } from 'typeorm';
+import { AssetEntity } from './asset.entity';
+
+@Entity('smart_info')
+export class SmartInfoEntity {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @Index({ unique: true })
+  @Column({ type: 'uuid' })
+  assetId: string;
+
+  @Column({ type: 'text', array: true, nullable: true })
+  tags: string[];
+
+  @Column({ type: 'text', array: true, nullable: true })
+  objects: string[];
+
+  @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
+  @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
+  asset: SmartInfoEntity;
+}

server/libs/database/src/entities/user-shared-album.entity.ts

@@ -0,0 +1,27 @@
+import { Column, Entity, JoinColumn, ManyToOne, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
+import { UserEntity } from './user.entity';
+import { SharedAlbumEntity } from './shared-album.entity';
+
+@Entity('user_shared_album')
+@Unique('PK_unique_user_in_album', ['albumId', 'sharedUserId'])
+export class UserSharedAlbumEntity {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @Column()
+  albumId: string;
+
+  @Column()
+  sharedUserId: string;
+
+  @ManyToOne(() => SharedAlbumEntity, (sharedAlbum) => sharedAlbum.sharedUsers, {
+    onDelete: 'CASCADE',
+    nullable: true,
+  })
+  @JoinColumn({ name: 'albumId' })
+  albumInfo: SharedAlbumEntity;
+
+  @ManyToOne(() => UserEntity)
+  @JoinColumn({ name: 'sharedUserId' })
+  userInfo: UserEntity;
+}

server/libs/database/src/entities/user.entity.ts

@@ -0,0 +1,34 @@
+import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn } from 'typeorm';
+
+@Entity('users')
+export class UserEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  firstName: string;
+
+  @Column()
+  lastName: string;
+
+  @Column()
+  isAdmin: boolean;
+
+  @Column()
+  email: string;
+
+  @Column({ select: false })
+  password: string;
+
+  @Column({ select: false })
+  salt: string;
+
+  @Column()
+  profileImagePath: string;
+
+  @Column()
+  isFirstLoggedIn: boolean;
+
+  @CreateDateColumn()
+  createdAt: string;
+}