chore(server) Add job for storage migration (#1117)

server/apps/microservices/src/processors/storage-migration.processor.ts

@@ -0,0 +1,61 @@
+import { APP_UPLOAD_LOCATION } from '@app/common';
+import { AssetEntity } from '@app/database/entities/asset.entity';
+import { ImmichConfigService } from '@app/immich-config';
+import { QueueNameEnum, templateMigrationProcessorName, updateTemplateProcessorName } from '@app/job';
+import { StorageService } from '@app/storage';
+import { Process, Processor } from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+
+@Processor(QueueNameEnum.STORAGE_MIGRATION)
+export class StorageMigrationProcessor {
+  readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
+
+  constructor(
+    private storageService: StorageService,
+    private immichConfigService: ImmichConfigService,
+
+    @InjectRepository(AssetEntity)
+    private assetRepository: Repository<AssetEntity>,
+  ) {}
+
+  /**
+   * Migration process when a new user set a new storage template.
+   * @param job
+   */
+  @Process({ name: templateMigrationProcessorName, concurrency: 100 })
+  async templateMigration() {
+    console.time('migrating-time');
+    const assets = await this.assetRepository.find({
+      relations: ['exifInfo'],
+    });
+
+    const livePhotoMap: Record<string, AssetEntity> = {};
+
+    for (const asset of assets) {
+      if (asset.livePhotoVideoId) {
+        livePhotoMap[asset.livePhotoVideoId] = asset;
+      }
+    }
+
+    for (const asset of assets) {
+      const livePhotoParentAsset = livePhotoMap[asset.id];
+      const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
+      await this.storageService.moveAsset(asset, filename);
+    }
+
+    await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
+    console.timeEnd('migrating-time');
+  }
+
+  /**
+   * Update config when a new storage template is set.
+   * This is to ensure the synchronization between processes.
+   * @param job
+   */
+  @Process({ name: updateTemplateProcessorName, concurrency: 1 })
+  async updateTemplate() {
+    await this.immichConfigService.refreshConfig();
+  }
+}

server/apps/microservices/src/processors/thumbnail.processor.ts

@@ -1,5 +1,4 @@
 import { APP_UPLOAD_LOCATION } from '@app/common';
-import { ImmichLogLevel } from '@app/common/constants/log-level.constant';
 import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
 import {
   WebpGeneratorProcessor,
@@ -11,7 +10,6 @@ import {
 } from '@app/job';
 import { InjectQueue, Process, Processor } from '@nestjs/bull';
 import { Logger } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
 import { InjectRepository } from '@nestjs/typeorm';
 import { mapAsset } from 'apps/immich/src/api-v1/asset/response-dto/asset-response.dto';
 import { Job, Queue } from 'bull';
@@ -27,7 +25,7 @@ import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interf
 
 @Processor(QueueNameEnum.THUMBNAIL_GENERATION)
 export class ThumbnailGeneratorProcessor {
-  private logLevel: ImmichLogLevel;
+  readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name);
 
   constructor(
     @InjectRepository(AssetEntity)
@@ -40,12 +38,7 @@ export class ThumbnailGeneratorProcessor {
 
     @InjectQueue(QueueNameEnum.MACHINE_LEARNING)
     private machineLearningQueue: Queue<IMachineLearningJob>,
-
-    private configService: ConfigService,
-  ) {
-    this.logLevel = this.configService.get('LOG_LEVEL') || ImmichLogLevel.SIMPLE;
-    // TODO - Add observable paterrn to listen to the config change
-  }
+  ) {}
 
   @Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 })
   async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
@@ -70,12 +63,8 @@ export class ThumbnailGeneratorProcessor {
           .rotate()
           .toFile(jpegThumbnailPath);
         await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
-      } catch (error) {
-        Logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id);
-
-        if (this.logLevel == ImmichLogLevel.VERBOSE) {
-          console.trace('Failed to generate jpeg thumbnail for asset', error);
-        }
+      } catch (error: any) {
+        this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
       }
 
       // Update resize path to send to generate webp queue
@@ -140,12 +129,8 @@ export class ThumbnailGeneratorProcessor {
     try {
       await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath);
       await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });
-    } catch (error) {
-      Logger.error('Failed to generate webp thumbnail for asset: ' + asset.id);
-
-      if (this.logLevel == ImmichLogLevel.VERBOSE) {
-        console.trace('Failed to generate webp thumbnail for asset', error);
-      }
+    } catch (error: any) {
+      this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
     }
   }
 }