refactor(server)*: tsconfigs (#2689)

* refactor(server): tsconfigs

* chore: dummy commit

* fix: start.sh

* chore: restore original entry scripts

server/src/domain/storage-template/index.ts

@@ -0,0 +1,2 @@
+export * from './storage-template.core';
+export * from './storage-template.service';

server/src/domain/storage-template/storage-template.core.ts

@@ -0,0 +1,160 @@
+import { AssetEntity, AssetType, SystemConfig } from '@app/infra/entities';
+import { Logger } from '@nestjs/common';
+import handlebar from 'handlebars';
+import * as luxon from 'luxon';
+import path from 'node:path';
+import sanitize from 'sanitize-filename';
+import { IStorageRepository, StorageCore } from '../storage';
+import {
+  ISystemConfigRepository,
+  supportedDayTokens,
+  supportedHourTokens,
+  supportedMinuteTokens,
+  supportedMonthTokens,
+  supportedSecondTokens,
+  supportedYearTokens,
+} from '../system-config';
+import { SystemConfigCore } from '../system-config/system-config.core';
+import { MoveAssetMetadata } from './storage-template.service';
+
+export class StorageTemplateCore {
+  private logger = new Logger(StorageTemplateCore.name);
+  private configCore: SystemConfigCore;
+  private storageTemplate: HandlebarsTemplateDelegate<any>;
+  private storageCore = new StorageCore();
+
+  constructor(
+    configRepository: ISystemConfigRepository,
+    config: SystemConfig,
+    private storageRepository: IStorageRepository,
+  ) {
+    this.storageTemplate = this.compile(config.storageTemplate.template);
+    this.configCore = new SystemConfigCore(configRepository);
+    this.configCore.addValidator((config) => this.validateConfig(config));
+    this.configCore.config$.subscribe((config) => this.onConfig(config));
+  }
+
+  public async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> {
+    const { storageLabel, filename } = metadata;
+
+    try {
+      const source = asset.originalPath;
+      const ext = path.extname(source).split('.').pop() as string;
+      const sanitized = sanitize(path.basename(filename, `.${ext}`));
+      const rootPath = this.storageCore.getLibraryFolder({ id: asset.ownerId, storageLabel });
+      const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
+      const fullPath = path.normalize(path.join(rootPath, storagePath));
+      let destination = `${fullPath}.${ext}`;
+
+      if (!fullPath.startsWith(rootPath)) {
+        this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
+        return source;
+      }
+
+      if (source === destination) {
+        return source;
+      }
+
+      /**
+       * In case of migrating duplicate filename to a new path, we need to check if it is already migrated
+       * Due to the mechanism of appending +1, +2, +3, etc to the filename
+       *
+       * Example:
+       * Source = upload/abc/def/FullSizeRender+7.heic
+       * Expected Destination = upload/abc/def/FullSizeRender.heic
+       *
+       * The file is already at the correct location, but since there are other FullSizeRender.heic files in the
+       * destination, it was renamed to FullSizeRender+7.heic.
+       *
+       * The lines below will be used to check if the differences between the source and destination is only the
+       * +7 suffix, and if so, it will be considered as already migrated.
+       */
+      if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) {
+        const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
+        const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
+        if (hasDuplicationAnnotation) {
+          return source;
+        }
+      }
+
+      let duplicateCount = 0;
+
+      while (true) {
+        const exists = await this.storageRepository.checkFileExists(destination);
+        if (!exists) {
+          break;
+        }
+
+        duplicateCount++;
+        destination = `${fullPath}+${duplicateCount}.${ext}`;
+      }
+
+      return destination;
+    } catch (error: any) {
+      this.logger.error(`Unable to get template path for ${filename}`, error);
+      return asset.originalPath;
+    }
+  }
+
+  private validateConfig(config: SystemConfig) {
+    this.validateStorageTemplate(config.storageTemplate.template);
+  }
+
+  private validateStorageTemplate(templateString: string) {
+    try {
+      const template = this.compile(templateString);
+      // test render an asset
+      this.render(
+        template,
+        {
+          fileCreatedAt: new Date(),
+          originalPath: '/upload/test/IMG_123.jpg',
+          type: AssetType.IMAGE,
+        } as AssetEntity,
+        'IMG_123',
+        'jpg',
+      );
+    } catch (e) {
+      this.logger.warn(`Storage template validation failed: ${JSON.stringify(e)}`);
+      throw new Error(`Invalid storage template: ${e}`);
+    }
+  }
+
+  private onConfig(config: SystemConfig) {
+    this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
+    this.storageTemplate = this.compile(config.storageTemplate.template);
+  }
+
+  private compile(template: string) {
+    return handlebar.compile(template, {
+      knownHelpers: undefined,
+      strict: true,
+    });
+  }
+
+  private render(template: HandlebarsTemplateDelegate<any>, asset: AssetEntity, filename: string, ext: string) {
+    const substitutions: Record<string, string> = {
+      filename,
+      ext,
+      filetype: asset.type == AssetType.IMAGE ? 'IMG' : 'VID',
+      filetypefull: asset.type == AssetType.IMAGE ? 'IMAGE' : 'VIDEO',
+    };
+
+    const dt = luxon.DateTime.fromJSDate(asset.fileCreatedAt);
+
+    const dateTokens = [
+      ...supportedYearTokens,
+      ...supportedMonthTokens,
+      ...supportedDayTokens,
+      ...supportedHourTokens,
+      ...supportedMinuteTokens,
+      ...supportedSecondTokens,
+    ];
+
+    for (const token of dateTokens) {
+      substitutions[token] = dt.toFormat(token);
+    }
+
+    return template(substitutions);
+  }
+}

server/src/domain/storage-template/storage-template.service.spec.ts

@@ -0,0 +1,198 @@
+import { when } from 'jest-when';
+import {
+  assetEntityStub,
+  newAssetRepositoryMock,
+  newStorageRepositoryMock,
+  newSystemConfigRepositoryMock,
+  newUserRepositoryMock,
+  systemConfigStub,
+  userEntityStub,
+} from '@test';
+import { IAssetRepository } from '../asset';
+import { StorageTemplateService } from '.';
+import { IStorageRepository } from '../storage/storage.repository';
+import { ISystemConfigRepository } from '../system-config';
+import { IUserRepository } from '../user';
+
+describe(StorageTemplateService.name, () => {
+  let sut: StorageTemplateService;
+  let assetMock: jest.Mocked<IAssetRepository>;
+  let configMock: jest.Mocked<ISystemConfigRepository>;
+  let storageMock: jest.Mocked<IStorageRepository>;
+  let userMock: jest.Mocked<IUserRepository>;
+
+  it('should work', () => {
+    expect(sut).toBeDefined();
+  });
+
+  beforeEach(async () => {
+    assetMock = newAssetRepositoryMock();
+    configMock = newSystemConfigRepositoryMock();
+    storageMock = newStorageRepositoryMock();
+    userMock = newUserRepositoryMock();
+
+    sut = new StorageTemplateService(assetMock, configMock, systemConfigStub.defaults, storageMock, userMock);
+  });
+
+  describe('handle template migration', () => {
+    it('should handle no assets', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [],
+        hasNextPage: false,
+      });
+      userMock.getList.mockResolvedValue([]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+    });
+
+    it('should handle an asset with a duplicate destination', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
+      assetMock.save.mockResolvedValue(assetEntityStub.image);
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      when(storageMock.checkFileExists)
+        .calledWith('upload/library/user-id/2023/2023-02-23/asset-id.ext')
+        .mockResolvedValue(true);
+
+      when(storageMock.checkFileExists)
+        .calledWith('upload/library/user-id/2023/2023-02-23/asset-id+1.ext')
+        .mockResolvedValue(false);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
+      });
+      expect(userMock.getList).toHaveBeenCalled();
+    });
+
+    it('should skip when an asset already matches the template', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [
+          {
+            ...assetEntityStub.image,
+            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
+          },
+        ],
+        hasNextPage: false,
+      });
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it('should skip when an asset is probably a duplicate', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [
+          {
+            ...assetEntityStub.image,
+            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
+          },
+        ],
+        hasNextPage: false,
+      });
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it('should move an asset', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
+      assetMock.save.mockResolvedValue(assetEntityStub.image);
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/library/user-id/2023/2023-02-23/asset-id.ext',
+      );
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
+      });
+    });
+
+    it('should use the user storage label', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
+      assetMock.save.mockResolvedValue(assetEntityStub.image);
+      userMock.getList.mockResolvedValue([userEntityStub.storageLabel]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/library/label-1/2023/2023-02-23/asset-id.ext',
+      );
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/library/label-1/2023/2023-02-23/asset-id.ext',
+      });
+    });
+
+    it('should not update the database if the move fails', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
+      storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/library/user-id/2023/2023-02-23/asset-id.ext',
+      );
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it('should move the asset back if the database fails', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
+      assetMock.save.mockRejectedValue('Connection Error!');
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetEntityStub.image.id,
+        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
+      });
+      expect(storageMock.moveFile.mock.calls).toEqual([
+        ['/original/path.ext', 'upload/library/user-id/2023/2023-02-23/asset-id.ext'],
+        ['upload/library/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'],
+      ]);
+    });
+  });
+});

server/src/domain/storage-template/storage-template.service.ts

@@ -0,0 +1,116 @@
+import { AssetEntity, SystemConfig } from '@app/infra/entities';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { IAssetRepository } from '../asset/asset.repository';
+import { APP_MEDIA_LOCATION } from '../domain.constant';
+import { getLivePhotoMotionFilename, usePagination } from '../domain.util';
+import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
+import { IStorageRepository } from '../storage/storage.repository';
+import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
+import { IUserRepository } from '../user/user.repository';
+import { StorageTemplateCore } from './storage-template.core';
+
+export interface MoveAssetMetadata {
+  storageLabel: string | null;
+  filename: string;
+}
+
+@Injectable()
+export class StorageTemplateService {
+  private logger = new Logger(StorageTemplateService.name);
+  private core: StorageTemplateCore;
+
+  constructor(
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
+    @Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
+    @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+    @Inject(IUserRepository) private userRepository: IUserRepository,
+  ) {
+    this.core = new StorageTemplateCore(configRepository, config, storageRepository);
+  }
+
+  async handleMigrationSingle({ id }: IEntityJob) {
+    const [asset] = await this.assetRepository.getByIds([id]);
+
+    const user = await this.userRepository.get(asset.ownerId);
+    const storageLabel = user?.storageLabel || null;
+    const filename = asset.originalFileName || asset.id;
+    await this.moveAsset(asset, { storageLabel, filename });
+
+    // move motion part of live photo
+    if (asset.livePhotoVideoId) {
+      const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]);
+      const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath);
+      await this.moveAsset(livePhotoVideo, { storageLabel, filename: motionFilename });
+    }
+
+    return true;
+  }
+
+  async handleMigration() {
+    try {
+      console.time('migrating-time');
+
+      const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+        this.assetRepository.getAll(pagination),
+      );
+      const users = await this.userRepository.getList();
+
+      for await (const assets of assetPagination) {
+        for (const asset of assets) {
+          const user = users.find((user) => user.id === asset.ownerId);
+          const storageLabel = user?.storageLabel || null;
+          const filename = asset.originalFileName || asset.id;
+          await this.moveAsset(asset, { storageLabel, filename });
+        }
+      }
+
+      this.logger.debug('Cleaning up empty directories...');
+      await this.storageRepository.removeEmptyDirs(APP_MEDIA_LOCATION);
+    } finally {
+      console.timeEnd('migrating-time');
+    }
+
+    return true;
+  }
+
+  // TODO: use asset core (once in domain)
+  async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
+    const destination = await this.core.getTemplatePath(asset, metadata);
+    if (asset.originalPath !== destination) {
+      const source = asset.originalPath;
+
+      let sidecarMoved = false;
+      try {
+        await this.storageRepository.moveFile(asset.originalPath, destination);
+
+        let sidecarDestination;
+        try {
+          if (asset.sidecarPath) {
+            sidecarDestination = `${destination}.xmp`;
+            await this.storageRepository.moveFile(asset.sidecarPath, sidecarDestination);
+            sidecarMoved = true;
+          }
+
+          await this.assetRepository.save({ id: asset.id, originalPath: destination, sidecarPath: sidecarDestination });
+          asset.originalPath = destination;
+          asset.sidecarPath = sidecarDestination || null;
+        } catch (error: any) {
+          this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
+
+          // Either sidecar move failed or the save failed. Eithr way, move media back
+          await this.storageRepository.moveFile(destination, source);
+
+          if (asset.sidecarPath && sidecarDestination && sidecarMoved) {
+            // If the sidecar was moved, that means the saved failed. So move both the sidecar and the
+            // media back into their original positions
+            await this.storageRepository.moveFile(sidecarDestination, asset.sidecarPath);
+          }
+        }
+      } catch (error: any) {
+        this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
+      }
+    }
+    return asset;
+  }
+}