feat(server): support for read-only assets and importing existing items in the filesystem (#2715)

* Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2025-10-29 17:40:28 +00:00 · 2023-06-21 22:33:20 -04:00
parent 7f44d508dc
commit e171fec5aa
55 changed files with 1321 additions and 128 deletions
--- a/server/src/domain/album/album.service.spec.ts
+++ b/server/src/domain/album/album.service.spec.ts
@@ -169,6 +169,7 @@ describe(AlbumService.name, () => {
          createdAt: new Date('2021-01-01'),
          deletedAt: null,
          updatedAt: new Date('2021-01-01'),
+          externalPath: null,
        },
        ownerId: 'admin_id',
        shared: false,
--- a/server/src/domain/api-key/api-key.core.ts
+++ b/server/src/domain/api-key/api-key.core.ts
@@ -19,6 +19,7 @@ export class APIKeyCore {
        isAdmin: user.isAdmin,
        isPublicUser: false,
        isAllowUpload: true,
+        externalPath: user.externalPath,
      };
    }

--- a/server/src/domain/auth/dto/auth-user.dto.ts
+++ b/server/src/domain/auth/dto/auth-user.dto.ts
@@ -8,4 +8,5 @@ export class AuthUserDto {
  isAllowDownload?: boolean;
  isShowExif?: boolean;
  accessTokenId?: string;
+  externalPath?: string | null;
 }
--- a/server/src/domain/crypto/crypto.repository.ts
+++ b/server/src/domain/crypto/crypto.repository.ts
@@ -2,6 +2,7 @@ export const ICryptoRepository = 'ICryptoRepository';

 export interface ICryptoRepository {
  randomBytes(size: number): Buffer;
+  hashFile(filePath: string): Promise<Buffer>;
  hashSha256(data: string): string;
  hashBcrypt(data: string | Buffer, saltOrRounds: string | number): Promise<string>;
  compareBcrypt(data: string | Buffer, encrypted: string): boolean;
--- a/server/src/domain/domain.constant.ts
+++ b/server/src/domain/domain.constant.ts
@@ -27,3 +27,60 @@ export function assertMachineLearningEnabled() {
    throw new BadRequestException('Machine learning is not enabled.');
  }
 }
+
+const validMimeTypes = [
+  'image/avif',
+  'image/gif',
+  'image/heic',
+  'image/heif',
+  'image/jpeg',
+  'image/jxl',
+  'image/png',
+  'image/tiff',
+  'image/webp',
+  'image/x-adobe-dng',
+  'image/x-arriflex-ari',
+  'image/x-canon-cr2',
+  'image/x-canon-cr3',
+  'image/x-canon-crw',
+  'image/x-epson-erf',
+  'image/x-fuji-raf',
+  'image/x-hasselblad-3fr',
+  'image/x-hasselblad-fff',
+  'image/x-kodak-dcr',
+  'image/x-kodak-k25',
+  'image/x-kodak-kdc',
+  'image/x-leica-rwl',
+  'image/x-minolta-mrw',
+  'image/x-nikon-nef',
+  'image/x-olympus-orf',
+  'image/x-olympus-ori',
+  'image/x-panasonic-raw',
+  'image/x-pentax-pef',
+  'image/x-phantom-cin',
+  'image/x-phaseone-cap',
+  'image/x-phaseone-iiq',
+  'image/x-samsung-srw',
+  'image/x-sigma-x3f',
+  'image/x-sony-arw',
+  'image/x-sony-sr2',
+  'image/x-sony-srf',
+  'video/3gpp',
+  'video/mp2t',
+  'video/mp4',
+  'video/mpeg',
+  'video/quicktime',
+  'video/webm',
+  'video/x-flv',
+  'video/x-matroska',
+  'video/x-ms-wmv',
+  'video/x-msvideo',
+];
+
+export function isSupportedFileType(mimetype: string): boolean {
+  return validMimeTypes.includes(mimetype);
+}
+
+export function isSidecarFileType(mimeType: string): boolean {
+  return ['application/xml', 'text/xml'].includes(mimeType);
+}
--- a/server/src/domain/partner/partner.service.spec.ts
+++ b/server/src/domain/partner/partner.service.spec.ts
@@ -17,6 +17,7 @@ const responseDto = {
    createdAt: new Date('2021-01-01'),
    deletedAt: null,
    updatedAt: new Date('2021-01-01'),
+    externalPath: null,
  },
  user1: {
    email: 'immich@test.com',
@@ -31,6 +32,7 @@ const responseDto = {
    createdAt: new Date('2021-01-01'),
    deletedAt: null,
    updatedAt: new Date('2021-01-01'),
+    externalPath: null,
  },
 };

--- a/server/src/domain/storage-template/storage-template.service.spec.ts
+++ b/server/src/domain/storage-template/storage-template.service.spec.ts
@@ -194,5 +194,26 @@ describe(StorageTemplateService.name, () => {
        ['upload/library/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'],
      ]);
    });
+
+    it('should not move read-only asset', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [
+          {
+            ...assetEntityStub.image,
+            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
+            isReadOnly: true,
+          },
+        ],
+        hasNextPage: false,
+      });
+      assetMock.save.mockResolvedValue(assetEntityStub.image);
+      userMock.getList.mockResolvedValue([userEntityStub.user1]);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
  });
 });
--- a/server/src/domain/storage-template/storage-template.service.ts
+++ b/server/src/domain/storage-template/storage-template.service.ts
@@ -76,6 +76,11 @@ export class StorageTemplateService {

  // TODO: use asset core (once in domain)
  async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
+    if (asset.isReadOnly) {
+      this.logger.verbose(`Not moving read-only asset: ${asset.originalPath}`);
+      return;
+    }
+
    const destination = await this.core.getTemplatePath(asset, metadata);
    if (asset.originalPath !== destination) {
      const source = asset.originalPath;
--- a/server/src/domain/user/dto/create-user.dto.ts
+++ b/server/src/domain/user/dto/create-user.dto.ts
@@ -23,6 +23,10 @@ export class CreateUserDto {
  @IsString()
  @Transform(toSanitized)
  storageLabel?: string | null;
+
+  @IsOptional()
+  @IsString()
+  externalPath?: string | null;
 }

 export class CreateAdminDto {
--- a/server/src/domain/user/dto/update-user.dto.ts
+++ b/server/src/domain/user/dto/update-user.dto.ts
@@ -29,6 +29,10 @@ export class UpdateUserDto {
  @Transform(toSanitized)
  storageLabel?: string;

+  @IsOptional()
+  @IsString()
+  externalPath?: string;
+
  @IsNotEmpty()
  @IsUUID('4')
  @ApiProperty({ format: 'uuid' })
--- a/server/src/domain/user/response-dto/user-response.dto.ts
+++ b/server/src/domain/user/response-dto/user-response.dto.ts
@@ -6,6 +6,7 @@ export class UserResponseDto {
  firstName!: string;
  lastName!: string;
  storageLabel!: string | null;
+  externalPath!: string | null;
  profileImagePath!: string;
  shouldChangePassword!: boolean;
  isAdmin!: boolean;
@@ -22,6 +23,7 @@ export function mapUser(entity: UserEntity): UserResponseDto {
    firstName: entity.firstName,
    lastName: entity.lastName,
    storageLabel: entity.storageLabel,
+    externalPath: entity.externalPath,
    profileImagePath: entity.profileImagePath,
    shouldChangePassword: entity.shouldChangePassword,
    isAdmin: entity.isAdmin,
--- a/server/src/domain/user/user.core.ts
+++ b/server/src/domain/user/user.core.ts
@@ -6,7 +6,6 @@ import {
  Logger,
  NotFoundException,
 } from '@nestjs/common';
-import { hash } from 'bcrypt';
 import { constants, createReadStream, ReadStream } from 'fs';
 import fs from 'fs/promises';
 import { AuthUserDto } from '../auth';
@@ -28,6 +27,7 @@ export class UserCore {
      // Users can never update the isAdmin property.
      delete dto.isAdmin;
      delete dto.storageLabel;
+      delete dto.externalPath;
    } else if (dto.isAdmin && authUser.id !== id) {
      // Admin cannot create another admin.
      throw new BadRequestException('The server already has an admin');
@@ -56,6 +56,10 @@ export class UserCore {
        dto.storageLabel = null;
      }

+      if (dto.externalPath === '') {
+        dto.externalPath = null;
+      }
+
      return this.userRepository.update(id, dto);
    } catch (e) {
      Logger.error(e, 'Failed to update user info');
@@ -79,7 +83,7 @@ export class UserCore {
    try {
      const payload: Partial<UserEntity> = { ...createUserDto };
      if (payload.password) {
-        payload.password = await hash(payload.password, SALT_ROUNDS);
+        payload.password = await this.cryptoRepository.hashBcrypt(payload.password, SALT_ROUNDS);
      }
      return this.userRepository.create(payload);
    } catch (e) {
--- a/server/src/domain/user/user.service.spec.ts
+++ b/server/src/domain/user/user.service.spec.ts
@@ -53,6 +53,7 @@ const adminUser: UserEntity = Object.freeze({
  tags: [],
  assets: [],
  storageLabel: 'admin',
+  externalPath: null,
 });

 const immichUser: UserEntity = Object.freeze({
@@ -71,6 +72,7 @@ const immichUser: UserEntity = Object.freeze({
  tags: [],
  assets: [],
  storageLabel: null,
+  externalPath: null,
 });

 const updatedImmichUser: UserEntity = Object.freeze({
@@ -89,6 +91,7 @@ const updatedImmichUser: UserEntity = Object.freeze({
  tags: [],
  assets: [],
  storageLabel: null,
+  externalPath: null,
 });

 const adminUserResponse = Object.freeze({
@@ -104,6 +107,7 @@ const adminUserResponse = Object.freeze({
  deletedAt: null,
  updatedAt: new Date('2021-01-01'),
  storageLabel: 'admin',
+  externalPath: null,
 });

 describe(UserService.name, () => {
@@ -153,6 +157,7 @@ describe(UserService.name, () => {
          deletedAt: null,
          updatedAt: new Date('2021-01-01'),
          storageLabel: 'admin',
+          externalPath: null,
        },
      ]);
    });