feat(server): Add support for client-side hashing (#2072)

* Modify controller DTOs

* Can check duplicates on server side

* Remove deviceassetid and deviceid

* Remove device ids from file uploader

* Add db migration for removed device ids

* Don't sanitize checksum

* Convert asset checksum to string

* Make checksum not optional for asset

* Use enums when rejecting duplicates

* Cleanup

* Return of the device id, but optional

* Don't use deviceId for upload folder

* Use checksum in thumb path

* Only use asset id in thumb path

* Openapi generation

* Put deviceAssetId back in asset response dto

* Add missing checksum in test fixture

* Add another missing checksum in test fixture

* Cleanup asset repository

* Add back previous /exists endpoint

* Require checksum to not be null

* Correctly set deviceId in db

* Remove index

* Fix compilation errors

* Make device id nullabel in asset response dto

* Reduce PR scope

* Revert asset service

* Reorder imports

* Reorder imports

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Update openapi

* Reduce PR scope

* refactor: asset bulk upload check

* chore: regenreate open-api

* chore: fix tests

* chore: tests

* update migrations and regenerate api

* Feat: use checksum in web file uploader

* Change to wasm-crypto

* Use crypto api for checksumming in web uploader

* Minor cleanup of file upload

* feat(web): pause and resume jobs

* Make device asset id not nullable again

* Cleanup

* Device id not nullable in response dto

* Update API specs

* Bump api specs

* Remove old TODO comment

* Remove NOT NULL constraint on checksum index

* Fix requested pubspec changes

* Remove unneeded import

* Update server/apps/immich/src/api-v1/asset/asset.service.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Update server/apps/immich/src/api-v1/asset/asset-repository.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Remove unneeded check

* Update server/apps/immich/src/api-v1/asset/asset-repository.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Remove hashing in the web uploader

* Cleanup file uploader

* Remove varchar from asset entity fields

* Return 200 from bulk upload check

* Put device asset id back into asset repository

* Merge migrations

* Revert pubspec lock

* Update openapi specs

* Merge upstream changes

* Fix failing asset service tests

* Fix formatting issue

* Cleanup migrations

* Remove newline from pubspec

* Revert newline

* Checkout main version

* Revert again

* Only return AssetCheck

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -10,13 +10,17 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
 import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
-import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
 import { In } from 'typeorm/find-options/operator/In';
 import { UpdateAssetDto } from './dto/update-asset.dto';
 import { ITagRepository } from '../tag/tag.repository';
 import { IsNull, Not } from 'typeorm';
 import { AssetSearchDto } from './dto/asset-search.dto';
 
+export interface AssetCheck {
+  id: string;
+  checksum: Buffer;
+}
+
 export interface IAssetRepository {
   get(id: string): Promise<AssetEntity | null>;
   create(
@@ -38,11 +42,8 @@ export interface IAssetRepository {
   getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
   getArchivedAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
   getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
-  getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
-  getExistingAssets(
-    userId: string,
-    checkDuplicateAssetDto: CheckExistingAssetsDto,
-  ): Promise<CheckExistingAssetsResponseDto>;
+  getAssetsByChecksums(userId: string, checksums: Buffer[]): Promise<AssetCheck[]>;
+  getExistingAssets(userId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise<string[]>;
   countByIdAndUser(assetId: string, userId: string): Promise<number>;
 }
 
@@ -310,41 +311,39 @@ export class AssetRepository implements IAssetRepository {
    * @returns Promise<string[]> - Array of assetIds belong to the device
    */
   async getAllByDeviceId(ownerId: string, deviceId: string): Promise<string[]> {
-    const rows = await this.assetRepository.find({
+    const items = await this.assetRepository.find({
+      select: { deviceAssetId: true },
       where: {
         ownerId,
         deviceId,
         isVisible: true,
       },
-      select: ['deviceAssetId'],
     });
-    const res: string[] = [];
-    rows.forEach((v) => res.push(v.deviceAssetId));
 
-    return res;
+    return items.map((asset) => asset.deviceAssetId);
   }
 
   /**
-   * Get asset by checksum on the database
+   * Get assets by checksums on the database
    * @param ownerId
-   * @param checksum
+   * @param checksums
    *
    */
-  getAssetByChecksum(ownerId: string, checksum: Buffer): Promise<AssetEntity> {
-    return this.assetRepository.findOneOrFail({
+  async getAssetsByChecksums(ownerId: string, checksums: Buffer[]): Promise<AssetCheck[]> {
+    return this.assetRepository.find({
+      select: {
+        id: true,
+        checksum: true,
+      },
       where: {
         ownerId,
-        checksum,
+        checksum: In(checksums),
       },
-      relations: ['exifInfo'],
     });
   }
 
-  async getExistingAssets(
-    ownerId: string,
-    checkDuplicateAssetDto: CheckExistingAssetsDto,
-  ): Promise<CheckExistingAssetsResponseDto> {
-    const existingAssets = await this.assetRepository.find({
+  async getExistingAssets(ownerId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise<string[]> {
+    const assets = await this.assetRepository.find({
       select: { deviceAssetId: true },
       where: {
         deviceAssetId: In(checkDuplicateAssetDto.deviceAssetIds),
@@ -352,7 +351,7 @@ export class AssetRepository implements IAssetRepository {
         ownerId,
       },
     });
-    return new CheckExistingAssetsResponseDto(existingAssets.map((a) => a.deviceAssetId));
+    return assets.map((asset) => asset.deviceAssetId);
   }
 
   async countByIdAndUser(assetId: string, ownerId: string): Promise<number> {

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -57,6 +57,8 @@ import { AssetSearchDto } from './dto/asset-search.dto';
 import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config';
 import FileNotEmptyValidator from '../validation/file-not-empty-validator';
 import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
+import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
+import { AssetBulkUploadCheckResponseDto } from './response-dto/asset-check-response.dto';
 import { AssetIdDto } from './dto/asset-id.dto';
 import { DeviceIdDto } from './dto/device-id.dto';
 
@@ -332,6 +334,19 @@ export class AssetController {
     return await this.assetService.checkExistingAssets(authUser, checkExistingAssetsDto);
   }
 
+  /**
+   * Checks if assets exist by checksums
+   */
+  @Authenticated()
+  @Post('/bulk-upload-check')
+  @HttpCode(200)
+  bulkUploadCheck(
+    @GetAuthUser() authUser: AuthUserDto,
+    @Body(ValidationPipe) dto: AssetBulkUploadCheckDto,
+  ): Promise<AssetBulkUploadCheckResponseDto> {
+    return this.assetService.bulkUploadCheck(authUser, dto);
+  }
+
   @Authenticated()
   @Post('/shared-link')
   async createAssetsSharedLink(

server/apps/immich/src/api-v1/asset/asset.core.ts

@@ -17,7 +17,7 @@ export class AssetCore {
       owner: { id: authUser.id } as UserEntity,
 
       mimeType: file.mimeType,
-      checksum: file.checksum || null,
+      checksum: file.checksum,
       originalPath: file.originalPath,
 
       deviceAssetId: dto.deviceAssetId,

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -157,7 +157,7 @@ describe('AssetService', () => {
       getLocationsByUserId: jest.fn(),
       getSearchPropertiesByUserId: jest.fn(),
       getAssetByTimeBucket: jest.fn(),
-      getAssetByChecksum: jest.fn(),
+      getAssetsByChecksums: jest.fn(),
       getAssetCountByUserId: jest.fn(),
       getArchivedAssetCountByUserId: jest.fn(),
       getExistingAssets: jest.fn(),
@@ -299,7 +299,7 @@ describe('AssetService', () => {
       (error as any).constraint = 'UQ_userid_checksum';
 
       assetRepositoryMock.create.mockRejectedValue(error);
-      assetRepositoryMock.getAssetByChecksum.mockResolvedValue(_getAsset_1());
+      assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([_getAsset_1()]);
 
       await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' });
 

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -63,6 +63,12 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
 import { AssetSearchDto } from './dto/asset-search.dto';
 import { AddAssetsDto } from '../album/dto/add-assets.dto';
 import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
+import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
+import {
+  AssetUploadAction,
+  AssetRejectReason,
+  AssetBulkUploadCheckResponseDto,
+} from './response-dto/asset-check-response.dto';
 
 const fileInfo = promisify(stat);
 
@@ -128,7 +134,8 @@ export class AssetService {
 
       // handle duplicates with a success response
       if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
-        const duplicate = await this.getAssetByChecksum(authUser.id, file.checksum);
+        const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum);
+        const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
         return { id: duplicate.id, duplicate: true };
       }
 
@@ -463,7 +470,40 @@ export class AssetService {
     authUser: AuthUserDto,
     checkExistingAssetsDto: CheckExistingAssetsDto,
   ): Promise<CheckExistingAssetsResponseDto> {
-    return this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto);
+    return {
+      existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto),
+    };
+  }
+
+  async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
+    const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
+    const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
+    const resultsMap: Record<string, string> = {};
+
+    for (const { id, checksum } of results) {
+      resultsMap[checksum.toString('hex')] = id;
+    }
+
+    return {
+      results: dto.assets.map(({ id, checksum }) => {
+        const duplicate = resultsMap[checksum];
+        if (duplicate) {
+          return {
+            id,
+            assetId: duplicate,
+            action: AssetUploadAction.REJECT,
+            reason: AssetRejectReason.DUPLICATE,
+          };
+        }
+
+        // TODO mime-check
+
+        return {
+          id,
+          action: AssetUploadAction.ACCEPT,
+        };
+      }),
+    };
   }
 
   async getAssetCountByTimeBucket(
@@ -482,10 +522,6 @@ export class AssetService {
     return mapAssetCountByTimeBucket(result);
   }
 
-  getAssetByChecksum(userId: string, checksum: Buffer) {
-    return this._assetRepository.getAssetByChecksum(userId, checksum);
-  }
-
   getAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
     return this._assetRepository.getAssetCountByUserId(authUser.id);
   }

server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts

@@ -0,0 +1,19 @@
+import { Type } from 'class-transformer';
+import { IsArray, IsNotEmpty, IsString, ValidateNested } from 'class-validator';
+
+export class AssetBulkUploadCheckItem {
+  @IsString()
+  @IsNotEmpty()
+  id!: string;
+
+  @IsString()
+  @IsNotEmpty()
+  checksum!: string;
+}
+
+export class AssetBulkUploadCheckDto {
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => AssetBulkUploadCheckItem)
+  assets!: AssetBulkUploadCheckItem[];
+}

server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts

@@ -0,0 +1,20 @@
+export class AssetBulkUploadCheckResult {
+  id!: string;
+  action!: AssetUploadAction;
+  reason?: AssetRejectReason;
+  assetId?: string;
+}
+
+export class AssetBulkUploadCheckResponseDto {
+  results!: AssetBulkUploadCheckResult[];
+}
+
+export enum AssetUploadAction {
+  ACCEPT = 'accept',
+  REJECT = 'reject',
+}
+
+export enum AssetRejectReason {
+  DUPLICATE = 'duplicate',
+  UNSUPPORTED_FORMAT = 'unsupported-format',
+}

server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts

@@ -1,6 +1,3 @@
 export class CheckExistingAssetsResponseDto {
-  constructor(existingIds: string[]) {
-    this.existingIds = existingIds;
-  }
-  existingIds: string[];
+  existingIds!: string[];
 }