feat(server) user-defined storage structure (#1098)

[Breaking] newly uploaded file will conform to the default structure of `{uploadLocation}/{userId}/year/year-month-day/filename.ext`

server/apps/immich/src/api-v1/asset/asset.module.ts

@@ -13,6 +13,7 @@ import { DownloadModule } from '../../modules/download/download.module';
 import { TagModule } from '../tag/tag.module';
 import { AlbumModule } from '../album/album.module';
 import { UserModule } from '../user/user.module';
+import { StorageModule } from '@app/storage';
 
 const ASSET_REPOSITORY_PROVIDER = {
   provide: ASSET_REPOSITORY,
@@ -28,6 +29,7 @@ const ASSET_REPOSITORY_PROVIDER = {
     UserModule,
     AlbumModule,
     TagModule,
+    StorageModule,
     forwardRef(() => AlbumModule),
     BullModule.registerQueue({
       name: QueueNameEnum.ASSET_UPLOADED,

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -11,7 +11,8 @@ import { DownloadService } from '../../modules/download/download.service';
 import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
 import { IAssetUploadedJob, IVideoTranscodeJob } from '@app/job';
 import { Queue } from 'bull';
-import { IAlbumRepository } from "../album/album-repository";
+import { IAlbumRepository } from '../album/album-repository';
+import { StorageService } from '@app/storage';
 
 describe('AssetService', () => {
   let sui: AssetService;
@@ -22,6 +23,7 @@ describe('AssetService', () => {
   let backgroundTaskServiceMock: jest.Mocked<BackgroundTaskService>;
   let assetUploadedQueueMock: jest.Mocked<Queue<IAssetUploadedJob>>;
   let videoConversionQueueMock: jest.Mocked<Queue<IVideoTranscodeJob>>;
+  let storageSeriveMock: jest.Mocked<StorageService>;
   const authUser: AuthUserDto = Object.freeze({
     id: 'user_id_1',
     email: 'auth@test.com',
@@ -139,6 +141,7 @@ describe('AssetService', () => {
       assetUploadedQueueMock,
       videoConversionQueueMock,
       downloadServiceMock as DownloadService,
+      storageSeriveMock,
     );
   });
 

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -55,6 +55,7 @@ import { Queue } from 'bull';
 import { DownloadService } from '../../modules/download/download.service';
 import { DownloadDto } from './dto/download-library.dto';
 import { ALBUM_REPOSITORY, IAlbumRepository } from '../album/album-repository';
+import { StorageService } from '@app/storage';
 
 const fileInfo = promisify(stat);
 
@@ -79,6 +80,8 @@ export class AssetService {
     private videoConversionQueue: Queue<IVideoTranscodeJob>,
 
     private downloadService: DownloadService,
+
+    private storageService: StorageService,
   ) {}
 
   public async handleUploadedAsset(
@@ -113,6 +116,8 @@ export class AssetService {
           throw new BadRequestException('Asset not created');
         }
 
+        await this.storageService.moveAsset(livePhotoAssetEntity, originalAssetData.originalname);
+
         await this.videoConversionQueue.add(
           mp4ConversionProcessorName,
           { asset: livePhotoAssetEntity },
@@ -139,13 +144,15 @@ export class AssetService {
         throw new BadRequestException('Asset not created');
       }
 
+      const movedAsset = await this.storageService.moveAsset(assetEntity, originalAssetData.originalname);
+
       await this.assetUploadedQueue.add(
         assetUploadedProcessorName,
-        { asset: assetEntity, fileName: originalAssetData.originalname },
-        { jobId: assetEntity.id },
+        { asset: movedAsset, fileName: originalAssetData.originalname },
+        { jobId: movedAsset.id },
       );
 
-      return new AssetFileUploadResponseDto(assetEntity.id);
+      return new AssetFileUploadResponseDto(movedAsset.id);
     } catch (err) {
       await this.backgroundTaskService.deleteFileOnDisk([
         {

server/apps/immich/src/api-v1/system-config/dto/system-config-storage-template.dto.ts

@@ -0,0 +1,7 @@
+import { IsNotEmpty, IsString } from 'class-validator';
+
+export class SystemConfigStorageTemplateDto {
+  @IsNotEmpty()
+  @IsString()
+  template!: string;
+}

server/apps/immich/src/api-v1/system-config/dto/system-config.dto.ts

@@ -2,6 +2,7 @@ import { SystemConfig } from '@app/database/entities/system-config.entity';
 import { ValidateNested } from 'class-validator';
 import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
 import { SystemConfigOAuthDto } from './system-config-oauth.dto';
+import { SystemConfigStorageTemplateDto } from './system-config-storage-template.dto';
 
 export class SystemConfigDto {
   @ValidateNested()
@@ -9,6 +10,9 @@ export class SystemConfigDto {
 
   @ValidateNested()
   oauth!: SystemConfigOAuthDto;
+
+  @ValidateNested()
+  storageTemplate!: SystemConfigStorageTemplateDto;
 }
 
 export function mapConfig(config: SystemConfig): SystemConfigDto {

server/apps/immich/src/api-v1/system-config/response-dto/system-config-template-storage-option.dto.ts

@@ -0,0 +1,9 @@
+export class SystemConfigTemplateStorageOptionDto {
+  yearOptions!: string[];
+  monthOptions!: string[];
+  dayOptions!: string[];
+  hourOptions!: string[];
+  minuteOptions!: string[];
+  secondOptions!: string[];
+  presetOptions!: string[];
+}

server/apps/immich/src/api-v1/system-config/system-config.controller.ts

@@ -1,6 +1,7 @@
 import { Body, Controller, Get, Put, ValidationPipe } from '@nestjs/common';
 import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
 import { Authenticated } from '../../decorators/authenticated.decorator';
+import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
 import { SystemConfigDto } from './dto/system-config.dto';
 import { SystemConfigService } from './system-config.service';
 
@@ -25,4 +26,9 @@ export class SystemConfigController {
   public updateConfig(@Body(ValidationPipe) dto: SystemConfigDto): Promise<SystemConfigDto> {
     return this.systemConfigService.updateConfig(dto);
   }
+
+  @Get('storage-template-options')
+  public getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto {
+    return this.systemConfigService.getStorageTemplateOptions();
+  }
 }

server/apps/immich/src/api-v1/system-config/system-config.service.ts

@@ -1,6 +1,16 @@
+import {
+  supportedDayTokens,
+  supportedHourTokens,
+  supportedMinuteTokens,
+  supportedMonthTokens,
+  supportedPresetTokens,
+  supportedSecondTokens,
+  supportedYearTokens,
+} from '@app/storage/constants/supported-datetime-template';
 import { Injectable } from '@nestjs/common';
 import { ImmichConfigService } from 'libs/immich-config/src';
 import { mapConfig, SystemConfigDto } from './dto/system-config.dto';
+import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
 
 @Injectable()
 export class SystemConfigService {
@@ -17,7 +27,21 @@ export class SystemConfigService {
   }
 
   public async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
-    await this.immichConfigService.updateConfig(dto);
-    return this.getConfig();
+    const config = await this.immichConfigService.updateConfig(dto);
+    return mapConfig(config);
+  }
+
+  public getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto {
+    const options = new SystemConfigTemplateStorageOptionDto();
+
+    options.dayOptions = supportedDayTokens;
+    options.monthOptions = supportedMonthTokens;
+    options.yearOptions = supportedYearTokens;
+    options.hourOptions = supportedHourTokens;
+    options.minuteOptions = supportedMinuteTokens;
+    options.secondOptions = supportedSecondTokens;
+    options.presetOptions = supportedPresetTokens;
+
+    return options;
   }
 }

server/apps/immich/src/api-v1/user/user.service.spec.ts

@@ -19,7 +19,7 @@ describe('UserService', () => {
     email: 'immich@test.com',
   });
 
-  const adminUser: UserEntity = Object.freeze({
+  const adminUser: UserEntity = {
     id: 'admin_id',
     email: 'admin@test.com',
     password: 'admin_password',
@@ -32,9 +32,9 @@ describe('UserService', () => {
     profileImagePath: '',
     createdAt: '2021-01-01',
     tags: [],
-  });
+  };
 
-  const immichUser: UserEntity = Object.freeze({
+  const immichUser: UserEntity = {
     id: 'immich_id',
     email: 'immich@test.com',
     password: 'immich_password',
@@ -47,9 +47,9 @@ describe('UserService', () => {
     profileImagePath: '',
     createdAt: '2021-01-01',
     tags: [],
-  });
+  };
 
-  const updatedImmichUser: UserEntity = Object.freeze({
+  const updatedImmichUser: UserEntity = {
     id: 'immich_id',
     email: 'immich@test.com',
     password: 'immich_password',
@@ -62,7 +62,7 @@ describe('UserService', () => {
     profileImagePath: '',
     createdAt: '2021-01-01',
     tags: [],
-  });
+  };
 
   beforeAll(() => {
     userRepositoryMock = newUserRepositoryMock();
@@ -75,7 +75,7 @@ describe('UserService', () => {
   });
 
   describe('Update user', () => {
-    it('should update user', () => {
+    it('should update user', async () => {
       const requestor = immichAuthUser;
       const userToUpdate = immichUser;
 
@@ -83,11 +83,11 @@ describe('UserService', () => {
       userRepositoryMock.get.mockImplementationOnce(() => Promise.resolve(userToUpdate));
       userRepositoryMock.update.mockImplementationOnce(() => Promise.resolve(updatedImmichUser));
 
-      const result = sui.updateUser(requestor, {
+      const result = await sui.updateUser(requestor, {
         id: userToUpdate.id,
         shouldChangePassword: true,
       });
-      expect(result).resolves.toBeDefined();
+      expect(result.shouldChangePassword).toEqual(true);
     });
 
     it('user can only update its information', () => {

server/apps/microservices/src/processors/thumbnail.processor.ts

@@ -44,6 +44,7 @@ export class ThumbnailGeneratorProcessor {
     private configService: ConfigService,
   ) {
     this.logLevel = this.configService.get('LOG_LEVEL') || ImmichLogLevel.SIMPLE;
+    // TODO - Add observable paterrn to listen to the config change
   }
 
   @Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 })
@@ -59,9 +60,7 @@ export class ThumbnailGeneratorProcessor {
       mkdirSync(resizePath, { recursive: true });
     }
 
-    const temp = asset.originalPath.split('/');
-    const originalFilename = temp[temp.length - 1].split('.')[0];
-    const jpegThumbnailPath = join(resizePath, `${originalFilename}.jpeg`);
+    const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
 
     if (asset.type == AssetType.IMAGE) {
       try {