refactor(server): remove checksum job (#1786)

server/apps/microservices/src/microservices.module.ts

@@ -14,9 +14,7 @@ import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module';
-import { MicroservicesService } from './microservices.service';
 import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
-import { GenerateChecksumProcessor } from './processors/generate-checksum.processor';
 import { MachineLearningProcessor } from './processors/machine-learning.processor';
 import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
 import { StorageMigrationProcessor } from './processors/storage-migration.processor';
@@ -46,12 +44,10 @@ import { DomainModule } from '@app/domain';
   ],
   controllers: [],
   providers: [
-    MicroservicesService,
     AssetUploadedProcessor,
     ThumbnailGeneratorProcessor,
     MetadataExtractionProcessor,
     VideoTranscodeProcessor,
-    GenerateChecksumProcessor,
     MachineLearningProcessor,
     UserDeletionProcessor,
     StorageMigrationProcessor,

server/apps/microservices/src/microservices.service.ts

@@ -1,16 +0,0 @@
-import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
-import { IJobRepository, JobName } from '@app/domain';
-
-const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(() => resolve(), ms));
-
-@Injectable()
-export class MicroservicesService implements OnModuleInit {
-  constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
-
-  async onModuleInit() {
-    // wait for migration
-    await sleep(10_000);
-
-    await this.jobRepository.add({ name: JobName.CHECKSUM_GENERATION });
-  }
-}

server/apps/microservices/src/processors/generate-checksum.processor.ts

@@ -1,82 +0,0 @@
-import { AssetEntity } from '@app/infra';
-import { JobName, QueueName } from '@app/domain';
-import { Process, Processor } from '@nestjs/bull';
-import { Logger } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
-import { createHash } from 'node:crypto';
-import fs from 'node:fs';
-import { FindOptionsWhere, IsNull, MoreThan, QueryFailedError, Repository } from 'typeorm';
-
-// TODO: just temporary task to generate previous uploaded assets.
-@Processor(QueueName.CHECKSUM_GENERATION)
-export class GenerateChecksumProcessor {
-  constructor(
-    @InjectRepository(AssetEntity)
-    private assetRepository: Repository<AssetEntity>,
-  ) {}
-
-  @Process(JobName.CHECKSUM_GENERATION)
-  async generateChecksum() {
-    const pageSize = 200;
-    let hasNext = true;
-    let lastErrAssetId: string | undefined = undefined;
-
-    while (hasNext) {
-      const whereStat: FindOptionsWhere<AssetEntity> = {
-        checksum: IsNull(),
-      };
-
-      if (lastErrAssetId) {
-        whereStat.id = MoreThan(lastErrAssetId);
-      }
-
-      const assets = await this.assetRepository.find({
-        where: whereStat,
-        take: pageSize,
-        order: { id: 'ASC' },
-      });
-
-      if (!assets?.length) {
-        hasNext = false; // avoid using break
-      } else {
-        for (const asset of assets) {
-          try {
-            await this.generateAssetChecksum(asset);
-          } catch (err: any) {
-            lastErrAssetId = asset.id;
-
-            if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') {
-              Logger.error(`${asset.originalPath} duplicated`);
-            } else {
-              Logger.error(`checksum generation ${err}`);
-            }
-          }
-        }
-
-        // break when reach to the last page
-        if (assets.length < pageSize) {
-          hasNext = false;
-        }
-      }
-    }
-
-    Logger.log(`checksum generation done!`);
-  }
-
-  private async generateAssetChecksum(asset: AssetEntity) {
-    if (!asset.originalPath) return;
-    if (!fs.existsSync(asset.originalPath)) return;
-
-    const fileReadStream = fs.createReadStream(asset.originalPath);
-    const sha1Hash = createHash('sha1');
-    const deferred = new Promise<Buffer>((resolve, reject) => {
-      sha1Hash.once('error', (err) => reject(err));
-      sha1Hash.once('finish', () => resolve(sha1Hash.read()));
-    });
-
-    fileReadStream.pipe(sha1Hash);
-    const checksum = await deferred;
-
-    await this.assetRepository.update(asset.id, { checksum });
-  }
-}