feat(server): Add support for client-side hashing (#2072)

* Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2026-01-09 02:35:49 +00:00 · 2023-05-24 23:08:21 +02:00
parent 49b74e9091
commit 1b54c4f8e7
33 changed files with 1396 additions and 58 deletions
--- a/server/apps/immich/src/api-v1/asset/asset-repository.ts
+++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts
@@ -10,13 +10,17 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
 import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
-import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
 import { In } from 'typeorm/find-options/operator/In';
 import { UpdateAssetDto } from './dto/update-asset.dto';
 import { ITagRepository } from '../tag/tag.repository';
 import { IsNull, Not } from 'typeorm';
 import { AssetSearchDto } from './dto/asset-search.dto';

+export interface AssetCheck {
+  id: string;
+  checksum: Buffer;
+}
+
 export interface IAssetRepository {
  get(id: string): Promise<AssetEntity | null>;
  create(
@@ -38,11 +42,8 @@ export interface IAssetRepository {
  getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
  getArchivedAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
  getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
-  getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
-  getExistingAssets(
-    userId: string,
-    checkDuplicateAssetDto: CheckExistingAssetsDto,
-  ): Promise<CheckExistingAssetsResponseDto>;
+  getAssetsByChecksums(userId: string, checksums: Buffer[]): Promise<AssetCheck[]>;
+  getExistingAssets(userId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise<string[]>;
  countByIdAndUser(assetId: string, userId: string): Promise<number>;
 }

@@ -310,41 +311,39 @@ export class AssetRepository implements IAssetRepository {
   * @returns Promise<string[]> - Array of assetIds belong to the device
   */
  async getAllByDeviceId(ownerId: string, deviceId: string): Promise<string[]> {
-    const rows = await this.assetRepository.find({
+    const items = await this.assetRepository.find({
+      select: { deviceAssetId: true },
      where: {
        ownerId,
        deviceId,
        isVisible: true,
      },
-      select: ['deviceAssetId'],
    });
-    const res: string[] = [];
-    rows.forEach((v) => res.push(v.deviceAssetId));

-    return res;
+    return items.map((asset) => asset.deviceAssetId);
  }

  /**
-   * Get asset by checksum on the database
+   * Get assets by checksums on the database
   * @param ownerId
-   * @param checksum
+   * @param checksums
   *
   */
-  getAssetByChecksum(ownerId: string, checksum: Buffer): Promise<AssetEntity> {
-    return this.assetRepository.findOneOrFail({
+  async getAssetsByChecksums(ownerId: string, checksums: Buffer[]): Promise<AssetCheck[]> {
+    return this.assetRepository.find({
+      select: {
+        id: true,
+        checksum: true,
+      },
      where: {
        ownerId,
-        checksum,
+        checksum: In(checksums),
      },
-      relations: ['exifInfo'],
    });
  }

-  async getExistingAssets(
-    ownerId: string,
-    checkDuplicateAssetDto: CheckExistingAssetsDto,
-  ): Promise<CheckExistingAssetsResponseDto> {
-    const existingAssets = await this.assetRepository.find({
+  async getExistingAssets(ownerId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise<string[]> {
+    const assets = await this.assetRepository.find({
      select: { deviceAssetId: true },
      where: {
        deviceAssetId: In(checkDuplicateAssetDto.deviceAssetIds),
@@ -352,7 +351,7 @@ export class AssetRepository implements IAssetRepository {
        ownerId,
      },
    });
-    return new CheckExistingAssetsResponseDto(existingAssets.map((a) => a.deviceAssetId));
+    return assets.map((asset) => asset.deviceAssetId);
  }

  async countByIdAndUser(assetId: string, ownerId: string): Promise<number> {
--- a/server/apps/immich/src/api-v1/asset/asset.controller.ts
+++ b/server/apps/immich/src/api-v1/asset/asset.controller.ts
@@ -57,6 +57,8 @@ import { AssetSearchDto } from './dto/asset-search.dto';
 import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config';
 import FileNotEmptyValidator from '../validation/file-not-empty-validator';
 import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
+import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
+import { AssetBulkUploadCheckResponseDto } from './response-dto/asset-check-response.dto';
 import { AssetIdDto } from './dto/asset-id.dto';
 import { DeviceIdDto } from './dto/device-id.dto';

@@ -332,6 +334,19 @@ export class AssetController {
    return await this.assetService.checkExistingAssets(authUser, checkExistingAssetsDto);
  }

+  /**
+   * Checks if assets exist by checksums
+   */
+  @Authenticated()
+  @Post('/bulk-upload-check')
+  @HttpCode(200)
+  bulkUploadCheck(
+    @GetAuthUser() authUser: AuthUserDto,
+    @Body(ValidationPipe) dto: AssetBulkUploadCheckDto,
+  ): Promise<AssetBulkUploadCheckResponseDto> {
+    return this.assetService.bulkUploadCheck(authUser, dto);
+  }
+
  @Authenticated()
  @Post('/shared-link')
  async createAssetsSharedLink(
--- a/server/apps/immich/src/api-v1/asset/asset.core.ts
+++ b/server/apps/immich/src/api-v1/asset/asset.core.ts
@@ -17,7 +17,7 @@ export class AssetCore {
      owner: { id: authUser.id } as UserEntity,

      mimeType: file.mimeType,
-      checksum: file.checksum || null,
+      checksum: file.checksum,
      originalPath: file.originalPath,

      deviceAssetId: dto.deviceAssetId,
--- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts
+++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts
@@ -157,7 +157,7 @@ describe('AssetService', () => {
      getLocationsByUserId: jest.fn(),
      getSearchPropertiesByUserId: jest.fn(),
      getAssetByTimeBucket: jest.fn(),
-      getAssetByChecksum: jest.fn(),
+      getAssetsByChecksums: jest.fn(),
      getAssetCountByUserId: jest.fn(),
      getArchivedAssetCountByUserId: jest.fn(),
      getExistingAssets: jest.fn(),
@@ -299,7 +299,7 @@ describe('AssetService', () => {
      (error as any).constraint = 'UQ_userid_checksum';

      assetRepositoryMock.create.mockRejectedValue(error);
-      assetRepositoryMock.getAssetByChecksum.mockResolvedValue(_getAsset_1());
+      assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([_getAsset_1()]);

      await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' });

--- a/server/apps/immich/src/api-v1/asset/asset.service.ts
+++ b/server/apps/immich/src/api-v1/asset/asset.service.ts
@@ -63,6 +63,12 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
 import { AssetSearchDto } from './dto/asset-search.dto';
 import { AddAssetsDto } from '../album/dto/add-assets.dto';
 import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
+import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
+import {
+  AssetUploadAction,
+  AssetRejectReason,
+  AssetBulkUploadCheckResponseDto,
+} from './response-dto/asset-check-response.dto';

 const fileInfo = promisify(stat);

@@ -128,7 +134,8 @@ export class AssetService {

      // handle duplicates with a success response
      if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
-        const duplicate = await this.getAssetByChecksum(authUser.id, file.checksum);
+        const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum);
+        const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
        return { id: duplicate.id, duplicate: true };
      }

@@ -463,7 +470,40 @@ export class AssetService {
    authUser: AuthUserDto,
    checkExistingAssetsDto: CheckExistingAssetsDto,
  ): Promise<CheckExistingAssetsResponseDto> {
-    return this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto);
+    return {
+      existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto),
+    };
+  }
+
+  async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
+    const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
+    const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
+    const resultsMap: Record<string, string> = {};
+
+    for (const { id, checksum } of results) {
+      resultsMap[checksum.toString('hex')] = id;
+    }
+
+    return {
+      results: dto.assets.map(({ id, checksum }) => {
+        const duplicate = resultsMap[checksum];
+        if (duplicate) {
+          return {
+            id,
+            assetId: duplicate,
+            action: AssetUploadAction.REJECT,
+            reason: AssetRejectReason.DUPLICATE,
+          };
+        }
+
+        // TODO mime-check
+
+        return {
+          id,
+          action: AssetUploadAction.ACCEPT,
+        };
+      }),
+    };
  }

  async getAssetCountByTimeBucket(
@@ -482,10 +522,6 @@ export class AssetService {
    return mapAssetCountByTimeBucket(result);
  }

-  getAssetByChecksum(userId: string, checksum: Buffer) {
-    return this._assetRepository.getAssetByChecksum(userId, checksum);
-  }
-
  getAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
    return this._assetRepository.getAssetCountByUserId(authUser.id);
  }
--- a/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts
+++ b/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts
@@ -0,0 +1,19 @@
+import { Type } from 'class-transformer';
+import { IsArray, IsNotEmpty, IsString, ValidateNested } from 'class-validator';
+
+export class AssetBulkUploadCheckItem {
+  @IsString()
+  @IsNotEmpty()
+  id!: string;
+
+  @IsString()
+  @IsNotEmpty()
+  checksum!: string;
+}
+
+export class AssetBulkUploadCheckDto {
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => AssetBulkUploadCheckItem)
+  assets!: AssetBulkUploadCheckItem[];
+}
--- a/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts
+++ b/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts
@@ -0,0 +1,20 @@
+export class AssetBulkUploadCheckResult {
+  id!: string;
+  action!: AssetUploadAction;
+  reason?: AssetRejectReason;
+  assetId?: string;
+}
+
+export class AssetBulkUploadCheckResponseDto {
+  results!: AssetBulkUploadCheckResult[];
+}
+
+export enum AssetUploadAction {
+  ACCEPT = 'accept',
+  REJECT = 'reject',
+}
+
+export enum AssetRejectReason {
+  DUPLICATE = 'duplicate',
+  UNSUPPORTED_FORMAT = 'unsupported-format',
+}
--- a/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts
+++ b/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts
@@ -1,6 +1,3 @@
 export class CheckExistingAssetsResponseDto {
-  constructor(existingIds: string[]) {
-    this.existingIds = existingIds;
-  }
-  existingIds: string[];
+  existingIds!: string[];
 }
--- a/server/immich-openapi-specs.json
+++ b/server/immich-openapi-specs.json
@@ -3251,6 +3251,49 @@
        ]
      }
    },
+    "/asset/bulk-upload-check": {
+      "post": {
+        "operationId": "bulkUploadCheck",
+        "description": "Checks if assets exist by checksums",
+        "parameters": [],
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/AssetBulkUploadCheckDto"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AssetBulkUploadCheckResponseDto"
+                }
+              }
+            }
+          }
+        },
+        "tags": [
+          "Asset"
+        ],
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ]
+      }
+    },
    "/asset/shared-link": {
      "post": {
        "operationId": "createAssetsSharedLink",
@@ -6046,6 +6089,78 @@
          "existingIds"
        ]
      },
+      "AssetBulkUploadCheckItem": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "checksum": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "id",
+          "checksum"
+        ]
+      },
+      "AssetBulkUploadCheckDto": {
+        "type": "object",
+        "properties": {
+          "assets": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/AssetBulkUploadCheckItem"
+            }
+          }
+        },
+        "required": [
+          "assets"
+        ]
+      },
+      "AssetBulkUploadCheckResult": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "action": {
+            "type": "string",
+            "enum": [
+              "accept",
+              "reject"
+            ]
+          },
+          "reason": {
+            "type": "string",
+            "enum": [
+              "duplicate",
+              "unsupported-format"
+            ]
+          },
+          "assetId": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "id",
+          "action"
+        ]
+      },
+      "AssetBulkUploadCheckResponseDto": {
+        "type": "object",
+        "properties": {
+          "results": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/AssetBulkUploadCheckResult"
+            }
+          }
+        },
+        "required": [
+          "results"
+        ]
+      },
      "CreateAssetsShareLinkDto": {
        "type": "object",
        "properties": {
--- a/server/libs/domain/test/fixtures.ts
+++ b/server/libs/domain/test/fixtures.ts
@@ -147,6 +147,7 @@ export const assetEntityStub = {
    deviceId: 'device-id',
    originalPath: 'upload/upload/path.ext',
    resizePath: null,
+    checksum: Buffer.from('file hash', 'utf8'),
    type: AssetType.IMAGE,
    webpPath: null,
    encodedVideoPath: null,
@@ -173,6 +174,7 @@ export const assetEntityStub = {
    deviceId: 'device-id',
    originalPath: '/original/path.ext',
    resizePath: '/uploads/user-id/thumbs/path.ext',
+    checksum: Buffer.from('file hash', 'utf8'),
    type: AssetType.IMAGE,
    webpPath: null,
    encodedVideoPath: null,
@@ -201,6 +203,7 @@ export const assetEntityStub = {
    deviceId: 'device-id',
    originalPath: '/original/path.ext',
    resizePath: '/uploads/user-id/thumbs/path.ext',
+    checksum: Buffer.from('file hash', 'utf8'),
    type: AssetType.VIDEO,
    webpPath: null,
    encodedVideoPath: null,
@@ -246,6 +249,7 @@ export const assetEntityStub = {
    owner: userEntityStub.user1,
    ownerId: 'user-id',
    deviceId: 'device-id',
+    checksum: Buffer.from('file hash', 'utf8'),
    originalPath: '/original/path.ext',
    resizePath: '/uploads/user-id/thumbs/path.ext',
    type: AssetType.IMAGE,
@@ -663,6 +667,7 @@ export const sharedLinkStub = {
          type: AssetType.VIDEO,
          originalPath: 'fake_path/jpeg',
          resizePath: '',
+          checksum: Buffer.from('file hash', 'utf8'),
          fileModifiedAt: today.toISOString(),
          fileCreatedAt: today.toISOString(),
          createdAt: today.toISOString(),
--- a/server/libs/infra/src/entities/asset.entity.ts
+++ b/server/libs/infra/src/entities/asset.entity.ts
@@ -75,9 +75,9 @@ export class AssetEntity {
  @Column({ type: 'varchar', nullable: true })
  mimeType!: string | null;

-  @Column({ type: 'bytea', nullable: true, select: false })
-  @Index({ where: `'checksum' IS NOT NULL` }) // avoid null index
-  checksum?: Buffer | null; // sha1 checksum
+  @Column({ type: 'bytea' })
+  @Index()
+  checksum!: Buffer; // sha1 checksum

  @Column({ type: 'varchar', nullable: true })
  duration!: string | null;
--- a/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts
+++ b/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts
@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class RequireChecksumNotNull1684328185099 implements MigrationInterface {
+  name = 'removeNotNullFromChecksumIndex1684328185099';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`);
+    await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" SET NOT NULL`);
+    await queryRunner.query(`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum") `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_8d3efe36c0755849395e6ea866"`);
+    await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" DROP NOT NULL`);
+    await queryRunner.query(
+      `CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE ('checksum' IS NOT NULL)`,
+    );
+  }
+}