feat(server): Add support for client-side hashing (#2072)

* Modify controller DTOs

* Can check duplicates on server side

* Remove deviceassetid and deviceid

* Remove device ids from file uploader

* Add db migration for removed device ids

* Don't sanitize checksum

* Convert asset checksum to string

* Make checksum not optional for asset

* Use enums when rejecting duplicates

* Cleanup

* Return of the device id, but optional

* Don't use deviceId for upload folder

* Use checksum in thumb path

* Only use asset id in thumb path

* Openapi generation

* Put deviceAssetId back in asset response dto

* Add missing checksum in test fixture

* Add another missing checksum in test fixture

* Cleanup asset repository

* Add back previous /exists endpoint

* Require checksum to not be null

* Correctly set deviceId in db

* Remove index

* Fix compilation errors

* Make device id nullabel in asset response dto

* Reduce PR scope

* Revert asset service

* Reorder imports

* Reorder imports

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Update openapi

* Reduce PR scope

* refactor: asset bulk upload check

* chore: regenreate open-api

* chore: fix tests

* chore: tests

* update migrations and regenerate api

* Feat: use checksum in web file uploader

* Change to wasm-crypto

* Use crypto api for checksumming in web uploader

* Minor cleanup of file upload

* feat(web): pause and resume jobs

* Make device asset id not nullable again

* Cleanup

* Device id not nullable in response dto

* Update API specs

* Bump api specs

* Remove old TODO comment

* Remove NOT NULL constraint on checksum index

* Fix requested pubspec changes

* Remove unneeded import

* Update server/apps/immich/src/api-v1/asset/asset.service.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Update server/apps/immich/src/api-v1/asset/asset-repository.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Remove unneeded check

* Update server/apps/immich/src/api-v1/asset/asset-repository.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Remove hashing in the web uploader

* Cleanup file uploader

* Remove varchar from asset entity fields

* Return 200 from bulk upload check

* Put device asset id back into asset repository

* Merge migrations

* Revert pubspec lock

* Update openapi specs

* Merge upstream changes

* Fix failing asset service tests

* Fix formatting issue

* Cleanup migrations

* Remove newline from pubspec

* Revert newline

* Checkout main version

* Revert again

* Only return AssetCheck

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

server/libs/domain/test/fixtures.ts

@@ -147,6 +147,7 @@ export const assetEntityStub = {
     deviceId: 'device-id',
     originalPath: 'upload/upload/path.ext',
     resizePath: null,
+    checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.IMAGE,
     webpPath: null,
     encodedVideoPath: null,
@@ -173,6 +174,7 @@ export const assetEntityStub = {
     deviceId: 'device-id',
     originalPath: '/original/path.ext',
     resizePath: '/uploads/user-id/thumbs/path.ext',
+    checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.IMAGE,
     webpPath: null,
     encodedVideoPath: null,
@@ -201,6 +203,7 @@ export const assetEntityStub = {
     deviceId: 'device-id',
     originalPath: '/original/path.ext',
     resizePath: '/uploads/user-id/thumbs/path.ext',
+    checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.VIDEO,
     webpPath: null,
     encodedVideoPath: null,
@@ -246,6 +249,7 @@ export const assetEntityStub = {
     owner: userEntityStub.user1,
     ownerId: 'user-id',
     deviceId: 'device-id',
+    checksum: Buffer.from('file hash', 'utf8'),
     originalPath: '/original/path.ext',
     resizePath: '/uploads/user-id/thumbs/path.ext',
     type: AssetType.IMAGE,
@@ -663,6 +667,7 @@ export const sharedLinkStub = {
           type: AssetType.VIDEO,
           originalPath: 'fake_path/jpeg',
           resizePath: '',
+          checksum: Buffer.from('file hash', 'utf8'),
           fileModifiedAt: today.toISOString(),
           fileCreatedAt: today.toISOString(),
           createdAt: today.toISOString(),

server/libs/infra/src/entities/asset.entity.ts

@@ -75,9 +75,9 @@ export class AssetEntity {
   @Column({ type: 'varchar', nullable: true })
   mimeType!: string | null;
 
-  @Column({ type: 'bytea', nullable: true, select: false })
-  @Index({ where: `'checksum' IS NOT NULL` }) // avoid null index
-  checksum?: Buffer | null; // sha1 checksum
+  @Column({ type: 'bytea' })
+  @Index()
+  checksum!: Buffer; // sha1 checksum
 
   @Column({ type: 'varchar', nullable: true })
   duration!: string | null;

server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class RequireChecksumNotNull1684328185099 implements MigrationInterface {
+  name = 'removeNotNullFromChecksumIndex1684328185099';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`);
+    await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" SET NOT NULL`);
+    await queryRunner.query(`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum") `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_8d3efe36c0755849395e6ea866"`);
+    await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" DROP NOT NULL`);
+    await queryRunner.query(
+      `CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE ('checksum' IS NOT NULL)`,
+    );
+  }
+}