refactor(server): jobs and processors (#1787)

* refactor: jobs and processors

* refactor: storage migration processor

* fix: tests

* fix: code warning

* chore: ignore coverage from infra

* fix: sync move asset logic between job core and asset core

* refactor: move error handling inside of catch

* refactor(server): job core into dedicated service calls

* refactor: smart info

* fix: tests

* chore: smart info tests

* refactor: use asset repository

* refactor: thumbnail processor

* chore: coverage reqs

server/libs/domain/src/storage-template/index.ts

@@ -0,0 +1,2 @@
+export * from './storage-template.core';
+export * from './storage-template.service';

server/libs/domain/src/storage-template/storage-template.core.ts

@@ -0,0 +1,162 @@
+import { APP_UPLOAD_LOCATION } from '@app/common';
+import {
+  IStorageRepository,
+  ISystemConfigRepository,
+  supportedDayTokens,
+  supportedHourTokens,
+  supportedMinuteTokens,
+  supportedMonthTokens,
+  supportedSecondTokens,
+  supportedYearTokens,
+} from '@app/domain';
+import { AssetEntity, AssetType, SystemConfig } from '@app/infra/db/entities';
+import { Logger } from '@nestjs/common';
+import handlebar from 'handlebars';
+import * as luxon from 'luxon';
+import path from 'node:path';
+import sanitize from 'sanitize-filename';
+import { SystemConfigCore } from '../system-config/system-config.core';
+
+export class StorageTemplateCore {
+  private logger = new Logger(StorageTemplateCore.name);
+  private configCore: SystemConfigCore;
+  private storageTemplate: HandlebarsTemplateDelegate<any>;
+
+  constructor(
+    configRepository: ISystemConfigRepository,
+    config: SystemConfig,
+    private storageRepository: IStorageRepository,
+  ) {
+    this.storageTemplate = this.compile(config.storageTemplate.template);
+    this.configCore = new SystemConfigCore(configRepository);
+    this.configCore.addValidator((config) => this.validateConfig(config));
+    this.configCore.config$.subscribe((config) => this.onConfig(config));
+  }
+
+  public async getTemplatePath(asset: AssetEntity, filename: string): Promise<string> {
+    try {
+      const source = asset.originalPath;
+      const ext = path.extname(source).split('.').pop() as string;
+      const sanitized = sanitize(path.basename(filename, `.${ext}`));
+      const rootPath = path.join(APP_UPLOAD_LOCATION, asset.ownerId);
+      const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
+      const fullPath = path.normalize(path.join(rootPath, storagePath));
+      let destination = `${fullPath}.${ext}`;
+
+      if (!fullPath.startsWith(rootPath)) {
+        this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
+        return source;
+      }
+
+      if (source === destination) {
+        return source;
+      }
+
+      /**
+       * In case of migrating duplicate filename to a new path, we need to check if it is already migrated
+       * Due to the mechanism of appending +1, +2, +3, etc to the filename
+       *
+       * Example:
+       * Source = upload/abc/def/FullSizeRender+7.heic
+       * Expected Destination = upload/abc/def/FullSizeRender.heic
+       *
+       * The file is already at the correct location, but since there are other FullSizeRender.heic files in the
+       * destination, it was renamed to FullSizeRender+7.heic.
+       *
+       * The lines below will be used to check if the differences between the source and destination is only the
+       * +7 suffix, and if so, it will be considered as already migrated.
+       */
+      if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) {
+        const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
+        const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
+        if (hasDuplicationAnnotation) {
+          return source;
+        }
+      }
+
+      let duplicateCount = 0;
+
+      while (true) {
+        const exists = await this.storageRepository.checkFileExists(destination);
+        if (!exists) {
+          break;
+        }
+
+        duplicateCount++;
+        destination = `${fullPath}+${duplicateCount}.${ext}`;
+      }
+
+      return destination;
+    } catch (error: any) {
+      this.logger.error(`Unable to get template path for ${filename}`, error);
+      return asset.originalPath;
+    }
+  }
+
+  private validateConfig(config: SystemConfig) {
+    this.validateStorageTemplate(config.storageTemplate.template);
+  }
+
+  private validateStorageTemplate(templateString: string) {
+    try {
+      const template = this.compile(templateString);
+      // test render an asset
+      this.render(
+        template,
+        {
+          fileCreatedAt: new Date().toISOString(),
+          originalPath: '/upload/test/IMG_123.jpg',
+          type: AssetType.IMAGE,
+        } as AssetEntity,
+        'IMG_123',
+        'jpg',
+      );
+    } catch (e) {
+      this.logger.warn(`Storage template validation failed: ${JSON.stringify(e)}`);
+      throw new Error(`Invalid storage template: ${e}`);
+    }
+  }
+
+  private onConfig(config: SystemConfig) {
+    this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
+    this.storageTemplate = this.compile(config.storageTemplate.template);
+  }
+
+  private compile(template: string) {
+    return handlebar.compile(template, {
+      knownHelpers: undefined,
+      strict: true,
+    });
+  }
+
+  private render(template: HandlebarsTemplateDelegate<any>, asset: AssetEntity, filename: string, ext: string) {
+    const substitutions: Record<string, string> = {
+      filename,
+      ext,
+    };
+
+    const fileType = asset.type == AssetType.IMAGE ? 'IMG' : 'VID';
+    const fileTypeFull = asset.type == AssetType.IMAGE ? 'IMAGE' : 'VIDEO';
+
+    const dt = luxon.DateTime.fromISO(new Date(asset.fileCreatedAt).toISOString());
+
+    const dateTokens = [
+      ...supportedYearTokens,
+      ...supportedMonthTokens,
+      ...supportedDayTokens,
+      ...supportedHourTokens,
+      ...supportedMinuteTokens,
+      ...supportedSecondTokens,
+    ];
+
+    for (const token of dateTokens) {
+      substitutions[token] = dt.toFormat(token);
+    }
+
+    // Support file type token
+    substitutions.filetype = fileType;
+    substitutions.filetypefull = fileTypeFull;
+
+    return template(substitutions);
+  }
+}

server/libs/domain/src/storage-template/storage-template.service.spec.ts

@@ -0,0 +1,149 @@
+import { when } from 'jest-when';
+import {
+  assetEntityStub,
+  newAssetRepositoryMock,
+  newStorageRepositoryMock,
+  newSystemConfigRepositoryMock,
+  systemConfigStub,
+} from '../../test';
+import { IAssetRepository } from '../asset';
+import { StorageTemplateService } from '../storage-template';
+import { IStorageRepository } from '../storage/storage.repository';
+import { ISystemConfigRepository } from '../system-config';
+
+describe(StorageTemplateService.name, () => {
+  let sut: StorageTemplateService;
+  let assetMock: jest.Mocked<IAssetRepository>;
+  let configMock: jest.Mocked<ISystemConfigRepository>;
+  let storageMock: jest.Mocked<IStorageRepository>;
+
+  it('should work', () => {
+    expect(sut).toBeDefined();
+  });
+
+  beforeEach(async () => {
+    assetMock = newAssetRepositoryMock();
+    configMock = newSystemConfigRepositoryMock();
+    storageMock = newStorageRepositoryMock();
+    sut = new StorageTemplateService(assetMock, configMock, systemConfigStub.defaults, storageMock);
+  });
+
+  describe('handle template migration', () => {
+    it('should handle no assets', async () => {
+      assetMock.getAll.mockResolvedValue([]);
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+    });
+
+    it('should handle an asset with a duplicate destination', async () => {
+      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.save.mockResolvedValue(assetEntityStub.image);
+
+      when(storageMock.checkFileExists)
+        .calledWith('upload/user-id/2023/2023-02-23/asset-id.ext')
+        .mockResolvedValue(true);
+
+      when(storageMock.checkFileExists)
+        .calledWith('upload/user-id/2023/2023-02-23/asset-id+1.ext')
+        .mockResolvedValue(false);
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext',
+      });
+    });
+
+    it('should skip when an asset already matches the template', async () => {
+      assetMock.getAll.mockResolvedValue([
+        {
+          ...assetEntityStub.image,
+          originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
+        },
+      ]);
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it('should skip when an asset is probably a duplicate', async () => {
+      assetMock.getAll.mockResolvedValue([
+        {
+          ...assetEntityStub.image,
+          originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext',
+        },
+      ]);
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it('should move an asset', async () => {
+      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.save.mockResolvedValue(assetEntityStub.image);
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/user-id/2023/2023-02-23/asset-id.ext',
+      );
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
+      });
+    });
+
+    it('should not update the database if the move fails', async () => {
+      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/user-id/2023/2023-02-23/asset-id.ext',
+      );
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it('should move the asset back if the database fails', async () => {
+      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.save.mockRejectedValue('Connection Error!');
+
+      await sut.handleTemplateMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
+      });
+      expect(storageMock.moveFile.mock.calls).toEqual([
+        ['/original/path.ext', 'upload/user-id/2023/2023-02-23/asset-id.ext'],
+        ['upload/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'],
+      ]);
+    });
+  });
+
+  it('should handle an error', async () => {
+    assetMock.getAll.mockResolvedValue([]);
+    storageMock.removeEmptyDirs.mockRejectedValue(new Error('Read only filesystem'));
+
+    await sut.handleTemplateMigration();
+  });
+});

server/libs/domain/src/storage-template/storage-template.service.ts

@@ -0,0 +1,73 @@
+import { APP_UPLOAD_LOCATION } from '@app/common';
+import { AssetEntity, SystemConfig } from '@app/infra/db/entities';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { IAssetRepository } from '../asset/asset.repository';
+import { IStorageRepository } from '../storage/storage.repository';
+import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
+import { StorageTemplateCore } from './storage-template.core';
+
+@Injectable()
+export class StorageTemplateService {
+  private logger = new Logger(StorageTemplateService.name);
+  private core: StorageTemplateCore;
+
+  constructor(
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
+    @Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
+    @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+  ) {
+    this.core = new StorageTemplateCore(configRepository, config, storageRepository);
+  }
+
+  async handleTemplateMigration() {
+    try {
+      console.time('migrating-time');
+      const assets = await this.assetRepository.getAll();
+
+      const livePhotoMap: Record<string, AssetEntity> = {};
+
+      for (const asset of assets) {
+        if (asset.livePhotoVideoId) {
+          livePhotoMap[asset.livePhotoVideoId] = asset;
+        }
+      }
+
+      for (const asset of assets) {
+        const livePhotoParentAsset = livePhotoMap[asset.id];
+        // TODO: remove livePhoto specific stuff once upload is fixed
+        const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
+        await this.moveAsset(asset, filename);
+      }
+
+      this.logger.debug('Cleaning up empty directories...');
+      await this.storageRepository.removeEmptyDirs(APP_UPLOAD_LOCATION);
+    } catch (error: any) {
+      this.logger.error('Error running template migration', error);
+    } finally {
+      console.timeEnd('migrating-time');
+    }
+  }
+
+  // TODO: use asset core (once in domain)
+  async moveAsset(asset: AssetEntity, originalName: string) {
+    const destination = await this.core.getTemplatePath(asset, originalName);
+    if (asset.originalPath !== destination) {
+      const source = asset.originalPath;
+
+      try {
+        await this.storageRepository.moveFile(asset.originalPath, destination);
+        try {
+          await this.assetRepository.save({ id: asset.id, originalPath: destination });
+          asset.originalPath = destination;
+        } catch (error: any) {
+          this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
+          await this.storageRepository.moveFile(destination, source);
+        }
+      } catch (error: any) {
+        this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
+      }
+    }
+    return asset;
+  }
+}