feat(server): split generated content into a separate folder (#2047)

* feat: organize media folders

* fix: tests

server/libs/domain/src/domain.constant.ts

@@ -17,7 +17,7 @@ export const serverVersion: IServerVersion = {
 
 export const SERVER_VERSION = `${serverVersion.major}.${serverVersion.minor}.${serverVersion.patch}`;
 
-export const APP_UPLOAD_LOCATION = './upload';
+export const APP_MEDIA_LOCATION = process.env.IMMICH_MEDIA_LOCATION || './upload';
 
 export const MACHINE_LEARNING_URL = process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003';
 export const MACHINE_LEARNING_ENABLED = MACHINE_LEARNING_URL !== 'false';

server/libs/domain/src/media/media.service.spec.ts

@@ -75,16 +75,15 @@ describe(MediaService.name, () => {
     it('should generate a thumbnail for an image', async () => {
       await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.image) });
 
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id');
-      expect(mediaMock.resize).toHaveBeenCalledWith(
-        '/original/path.ext',
-        'upload/user-id/thumb/device-id/asset-id.jpeg',
-        { size: 1440, format: 'jpeg' },
-      );
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
+        size: 1440,
+        format: 'jpeg',
+      });
       expect(mediaMock.extractThumbnailFromExif).not.toHaveBeenCalled();
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
-        resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg',
+        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
       });
     });
 
@@ -93,33 +92,32 @@ describe(MediaService.name, () => {
 
       await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.image) });
 
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id');
-      expect(mediaMock.resize).toHaveBeenCalledWith(
-        '/original/path.ext',
-        'upload/user-id/thumb/device-id/asset-id.jpeg',
-        { size: 1440, format: 'jpeg' },
-      );
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
+        size: 1440,
+        format: 'jpeg',
+      });
       expect(mediaMock.extractThumbnailFromExif).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/user-id/thumb/device-id/asset-id.jpeg',
+        'upload/thumbs/user-id/asset-id.jpeg',
       );
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
-        resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg',
+        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
       });
     });
 
     it('should generate a thumbnail for a video', async () => {
       await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.video) });
 
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
       expect(mediaMock.extractVideoThumbnail).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/user-id/thumb/device-id/asset-id.jpeg',
+        'upload/thumbs/user-id/asset-id.jpeg',
       );
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
-        resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg',
+        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
       });
     });
 

server/libs/domain/src/media/media.service.ts

@@ -1,17 +1,16 @@
 import { AssetType } from '@app/infra/db/entities';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { join } from 'path';
-import sanitize from 'sanitize-filename';
 import { IAssetRepository, mapAsset, WithoutProperty } from '../asset';
 import { CommunicationEvent, ICommunicationRepository } from '../communication';
-import { APP_UPLOAD_LOCATION } from '../domain.constant';
 import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job';
-import { IStorageRepository } from '../storage';
+import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
 import { IMediaRepository } from './media.repository';
 
 @Injectable()
 export class MediaService {
   private logger = new Logger(MediaService.name);
+  private storageCore = new StorageCore();
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -41,11 +40,9 @@ export class MediaService {
     const { asset } = data;
 
     try {
-      const basePath = APP_UPLOAD_LOCATION;
-      const sanitizedDeviceId = sanitize(String(asset.deviceId));
-      const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId);
-      const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
+      const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
       this.storageRepository.mkdirSync(resizePath);
+      const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
 
       if (asset.type == AssetType.IMAGE) {
         try {

server/libs/domain/src/server-info/server-info.service.ts

@@ -1,5 +1,5 @@
 import { Inject, Injectable } from '@nestjs/common';
-import { APP_UPLOAD_LOCATION, serverVersion } from '../domain.constant';
+import { APP_MEDIA_LOCATION, serverVersion } from '../domain.constant';
 import { asHumanReadable } from '../domain.util';
 import { IStorageRepository } from '../storage';
 import { IUserRepository, UserStatsQueryResponse } from '../user';
@@ -13,7 +13,7 @@ export class ServerInfoService {
   ) {}
 
   async getInfo(): Promise<ServerInfoResponseDto> {
-    const diskInfo = await this.storageRepository.checkDiskUsage(APP_UPLOAD_LOCATION);
+    const diskInfo = await this.storageRepository.checkDiskUsage(APP_MEDIA_LOCATION);
 
     const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2);
 

server/libs/domain/src/storage-template/storage-template.core.ts

@@ -1,5 +1,11 @@
+import { AssetEntity, AssetType, SystemConfig } from '@app/infra/db/entities';
+import { Logger } from '@nestjs/common';
+import handlebar from 'handlebars';
+import * as luxon from 'luxon';
+import path from 'node:path';
+import sanitize from 'sanitize-filename';
+import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
 import {
-  IStorageRepository,
   ISystemConfigRepository,
   supportedDayTokens,
   supportedHourTokens,
@@ -7,20 +13,14 @@ import {
   supportedMonthTokens,
   supportedSecondTokens,
   supportedYearTokens,
-} from '@app/domain';
-import { AssetEntity, AssetType, SystemConfig } from '@app/infra/db/entities';
-import { Logger } from '@nestjs/common';
-import handlebar from 'handlebars';
-import * as luxon from 'luxon';
-import path from 'node:path';
-import sanitize from 'sanitize-filename';
-import { APP_UPLOAD_LOCATION } from '../domain.constant';
+} from '../system-config';
 import { SystemConfigCore } from '../system-config/system-config.core';
 
 export class StorageTemplateCore {
   private logger = new Logger(StorageTemplateCore.name);
   private configCore: SystemConfigCore;
   private storageTemplate: HandlebarsTemplateDelegate<any>;
+  private storageCore = new StorageCore();
 
   constructor(
     configRepository: ISystemConfigRepository,
@@ -38,7 +38,7 @@ export class StorageTemplateCore {
       const source = asset.originalPath;
       const ext = path.extname(source).split('.').pop() as string;
       const sanitized = sanitize(path.basename(filename, `.${ext}`));
-      const rootPath = path.join(APP_UPLOAD_LOCATION, asset.ownerId);
+      const rootPath = this.storageCore.getFolderLocation(StorageFolder.LIBRARY, asset.ownerId);
       const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
       const fullPath = path.normalize(path.join(rootPath, storagePath));
       let destination = `${fullPath}.${ext}`;

server/libs/domain/src/storage-template/storage-template.service.spec.ts

@@ -42,11 +42,11 @@ describe(StorageTemplateService.name, () => {
       assetMock.save.mockResolvedValue(assetEntityStub.image);
 
       when(storageMock.checkFileExists)
-        .calledWith('upload/user-id/2023/2023-02-23/asset-id.ext')
+        .calledWith('upload/library/user-id/2023/2023-02-23/asset-id.ext')
         .mockResolvedValue(true);
 
       when(storageMock.checkFileExists)
-        .calledWith('upload/user-id/2023/2023-02-23/asset-id+1.ext')
+        .calledWith('upload/library/user-id/2023/2023-02-23/asset-id+1.ext')
         .mockResolvedValue(false);
 
       await sut.handleTemplateMigration();
@@ -55,7 +55,7 @@ describe(StorageTemplateService.name, () => {
       expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
       expect(assetMock.save).toHaveBeenCalledWith({
         id: assetEntityStub.image.id,
-        originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext',
+        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
       });
     });
 
@@ -63,7 +63,7 @@ describe(StorageTemplateService.name, () => {
       assetMock.getAll.mockResolvedValue([
         {
           ...assetEntityStub.image,
-          originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
+          originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
         },
       ]);
 
@@ -79,7 +79,7 @@ describe(StorageTemplateService.name, () => {
       assetMock.getAll.mockResolvedValue([
         {
           ...assetEntityStub.image,
-          originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext',
+          originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
         },
       ]);
 
@@ -100,11 +100,11 @@ describe(StorageTemplateService.name, () => {
       expect(assetMock.getAll).toHaveBeenCalled();
       expect(storageMock.moveFile).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/user-id/2023/2023-02-23/asset-id.ext',
+        'upload/library/user-id/2023/2023-02-23/asset-id.ext',
       );
       expect(assetMock.save).toHaveBeenCalledWith({
         id: assetEntityStub.image.id,
-        originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
+        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
       });
     });
 
@@ -117,7 +117,7 @@ describe(StorageTemplateService.name, () => {
       expect(assetMock.getAll).toHaveBeenCalled();
       expect(storageMock.moveFile).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/user-id/2023/2023-02-23/asset-id.ext',
+        'upload/library/user-id/2023/2023-02-23/asset-id.ext',
       );
       expect(assetMock.save).not.toHaveBeenCalled();
     });
@@ -131,11 +131,11 @@ describe(StorageTemplateService.name, () => {
       expect(assetMock.getAll).toHaveBeenCalled();
       expect(assetMock.save).toHaveBeenCalledWith({
         id: assetEntityStub.image.id,
-        originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext',
+        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
       });
       expect(storageMock.moveFile.mock.calls).toEqual([
-        ['/original/path.ext', 'upload/user-id/2023/2023-02-23/asset-id.ext'],
-        ['upload/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'],
+        ['/original/path.ext', 'upload/library/user-id/2023/2023-02-23/asset-id.ext'],
+        ['upload/library/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'],
       ]);
     });
   });

server/libs/domain/src/storage-template/storage-template.service.ts

@@ -1,7 +1,7 @@
 import { AssetEntity, SystemConfig } from '@app/infra/db/entities';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { IAssetRepository } from '../asset/asset.repository';
-import { APP_UPLOAD_LOCATION } from '../domain.constant';
+import { APP_MEDIA_LOCATION } from '../domain.constant';
 import { IStorageRepository } from '../storage/storage.repository';
 import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
 import { StorageTemplateCore } from './storage-template.core';
@@ -41,7 +41,7 @@ export class StorageTemplateService {
       }
 
       this.logger.debug('Cleaning up empty directories...');
-      await this.storageRepository.removeEmptyDirs(APP_UPLOAD_LOCATION);
+      await this.storageRepository.removeEmptyDirs(APP_MEDIA_LOCATION);
     } catch (error: any) {
       this.logger.error('Error running template migration', error);
     } finally {

server/libs/domain/src/storage/index.ts

@@ -1,2 +1,3 @@
+export * from './storage.core';
 export * from './storage.repository';
 export * from './storage.service';

server/libs/domain/src/storage/storage.core.ts

@@ -0,0 +1,16 @@
+import { join } from 'node:path';
+import { APP_MEDIA_LOCATION } from '../domain.constant';
+
+export enum StorageFolder {
+  ENCODED_VIDEO = 'encoded-video',
+  LIBRARY = 'library',
+  UPLOAD = 'upload',
+  PROFILE = 'profile',
+  THUMBNAILS = 'thumbs',
+}
+
+export class StorageCore {
+  getFolderLocation(folder: StorageFolder, userId: string) {
+    return join(APP_MEDIA_LOCATION, folder, userId);
+  }
+}

server/libs/domain/src/user/user.service.spec.ts

@@ -467,7 +467,13 @@ describe(UserService.name, () => {
 
       await sut.handleUserDelete({ user });
 
-      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/deleted-user', { force: true, recursive: true });
+      const options = { force: true, recursive: true };
+
+      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/library/deleted-user', options);
+      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/upload/deleted-user', options);
+      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/profile/deleted-user', options);
+      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/thumbs/deleted-user', options);
+      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/encoded-video/deleted-user', options);
       expect(tokenMock.deleteAll).toHaveBeenCalledWith(user.id);
       expect(keyMock.deleteAll).toHaveBeenCalledWith(user.id);
       expect(albumMock.deleteAll).toHaveBeenCalledWith(user.id);

server/libs/domain/src/user/user.service.ts

@@ -2,14 +2,13 @@ import { UserEntity } from '@app/infra/db/entities';
 import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
 import { randomBytes } from 'crypto';
 import { ReadStream } from 'fs';
-import { join } from 'path';
 import { IAlbumRepository } from '../album/album.repository';
 import { IKeyRepository } from '../api-key/api-key.repository';
 import { IAssetRepository } from '../asset/asset.repository';
 import { AuthUserDto } from '../auth';
 import { ICryptoRepository } from '../crypto/crypto.repository';
-import { APP_UPLOAD_LOCATION } from '../domain.constant';
 import { IJobRepository, IUserDeletionJob, JobName } from '../job';
+import { StorageCore, StorageFolder } from '../storage';
 import { IStorageRepository } from '../storage/storage.repository';
 import { IUserTokenRepository } from '../user-token/user-token.repository';
 import { IUserRepository } from '../user/user.repository';
@@ -28,6 +27,8 @@ import { UserCore } from './user.core';
 export class UserService {
   private logger = new Logger(UserService.name);
   private userCore: UserCore;
+  private storageCore = new StorageCore();
+
   constructor(
     @Inject(IUserRepository) private userRepository: IUserRepository,
     @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@@ -162,9 +163,18 @@ export class UserService {
     this.logger.log(`Deleting user: ${user.id}`);
 
     try {
-      const userAssetDir = join(APP_UPLOAD_LOCATION, user.id);
-      this.logger.warn(`Removing user from filesystem: ${userAssetDir}`);
-      await this.storageRepository.unlinkDir(userAssetDir, { recursive: true, force: true });
+      const folders = [
+        this.storageCore.getFolderLocation(StorageFolder.LIBRARY, user.id),
+        this.storageCore.getFolderLocation(StorageFolder.UPLOAD, user.id),
+        this.storageCore.getFolderLocation(StorageFolder.PROFILE, user.id),
+        this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, user.id),
+        this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, user.id),
+      ];
+
+      for (const folder of folders) {
+        this.logger.warn(`Removing user from filesystem: ${folder}`);
+        await this.storageRepository.unlinkDir(folder, { recursive: true, force: true });
+      }
 
       this.logger.warn(`Removing user from database: ${user.id}`);
 

server/libs/domain/test/fixtures.ts

@@ -119,7 +119,7 @@ export const assetEntityStub = {
     owner: userEntityStub.user1,
     ownerId: 'user-id',
     deviceId: 'device-id',
-    originalPath: '/original/path.ext',
+    originalPath: 'upload/upload/path.ext',
     resizePath: null,
     type: AssetType.IMAGE,
     webpPath: null,