feat(server): support for read-only assets and importing existing items in the filesystem (#2715)

* Added read-only flag for assets, endpoint to trigger file import vs upload

* updated fixtures with new property

* if upload is 'read-only', ensure there is no existing asset at the designated originalPath

* added test for file import as well as detecting existing image at read-only destination location

* Added storage service test for a case where it should not move read-only assets

* upload doesn't need the read-only flag available, just importing

* default isReadOnly on import endpoint to true

* formatting fixes

* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation

* updated code to reflect changes in MR

* fixed read stream promise return type

* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates

* refactor: import asset

* chore: open api

* chore: tests

* Added externalPath support for individual users, updated UI to allow this to be set by admin

* added missing var for externalPath in ui

* chore: open api

* fix: compilation issues

* fix: server test

* built api, fixed user-response dto to include externalPath

* reverted accidental commit

* bad commit of duplicate externalPath in user response  dto

* fixed tests to include externalPath on expected result

* fix: unit tests

* centralized supported filetypes, perform file type checking of asset and sidecar during file import process

* centralized supported filetype check method to keep regex DRY

* fixed typo

* combined migrations into one

* update api

* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not

* update mimetype

* Fixed detect correct mimetype

* revert asset-upload config

* reverted domain.constant

* refactor

* fix mime-type issue

* fix format

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/infra/entities/asset.entity.ts

@@ -42,7 +42,7 @@ export class AssetEntity {
   @Column()
   type!: AssetType;
 
-  @Column()
+  @Column({ unique: true })
   originalPath!: string;
 
   @Column({ type: 'varchar', nullable: true })
@@ -75,6 +75,9 @@ export class AssetEntity {
   @Column({ type: 'boolean', default: false })
   isArchived!: boolean;
 
+  @Column({ type: 'boolean', default: false })
+  isReadOnly!: boolean;
+
   @Column({ type: 'varchar', nullable: true })
   mimeType!: string | null;
 

server/src/infra/entities/user.entity.ts

@@ -30,6 +30,9 @@ export class UserEntity {
   @Column({ type: 'varchar', unique: true, default: null })
   storageLabel!: string | null;
 
+  @Column({ type: 'varchar', default: null })
+  externalPath!: string | null;
+
   @Column({ default: '', select: false })
   password?: string;
 

server/src/infra/migrations/1686584273471-ImportAsset.ts

@@ -0,0 +1,18 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class ImportAsset1686584273471 implements MigrationInterface {
+    name = 'ImportAsset1686584273471'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "assets" ADD "isReadOnly" boolean NOT NULL DEFAULT false`);
+        await queryRunner.query(`ALTER TABLE "assets" ADD CONSTRAINT "UQ_4ed4f8052685ff5b1e7ca1058ba" UNIQUE ("originalPath")`);
+        await queryRunner.query(`ALTER TABLE "users" ADD "externalPath" character varying`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "assets" DROP CONSTRAINT "UQ_4ed4f8052685ff5b1e7ca1058ba"`);
+        await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "isReadOnly"`);
+        await queryRunner.query(`ALTER TABLE "users" DROP COLUMN "externalPath"`);
+    }
+
+}

server/src/infra/repositories/crypto.repository.ts

@@ -2,6 +2,7 @@ import { ICryptoRepository } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { compareSync, hash } from 'bcrypt';
 import { createHash, randomBytes } from 'crypto';
+import { createReadStream } from 'fs';
 
 @Injectable()
 export class CryptoRepository implements ICryptoRepository {
@@ -13,4 +14,14 @@ export class CryptoRepository implements ICryptoRepository {
   hashSha256(value: string) {
     return createHash('sha256').update(value).digest('base64');
   }
+
+  hashFile(filepath: string): Promise<Buffer> {
+    return new Promise<Buffer>((resolve, reject) => {
+      const hash = createHash('sha1');
+      const stream = createReadStream(filepath);
+      stream.on('error', (err) => reject(err));
+      stream.on('data', (chunk) => hash.update(chunk));
+      stream.on('end', () => resolve(hash.digest()));
+    });
+  }
 }