feat(server): xmp sidecar metadata (#2466)

* initial commit for XMP sidecar support

* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards

* didn't mean to commit default log level during testing

* new sidecar logic for video metadata as well

* Added xml mimetype for sidecars only

* don't need capture group for this regex

* wrong default value reverted

* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway

* simplified setter logic

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

* simplified logic per suggestions

* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing

* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar

* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync

* simplified logic of filename extraction and asset instantiation

* not sure how that got deleted..

* updated code per suggestions and comments in the PR

* stat was not being used, removed the variable set

* better type checking, using in-scope variables for exif getter instead of passing in every time

* removed commented out test

* ran and resolved all lints, formats, checks, and tests

* resolved suggested change in PR

* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function  for better type checking

* better error handling and moving files back to positions on move or save failure

* regenerated api

* format fixes

* Added XMP documentation

* documentation typo

* Merged in main

* missed merge conflict

* more changes due to a merge

* Resolving conflicts

* added icon for sidecar jobs

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/libs/domain/src/asset/asset.repository.ts

@@ -30,6 +30,11 @@ export enum WithoutProperty {
   CLIP_ENCODING = 'clip-embedding',
   OBJECT_TAGS = 'object-tags',
   FACES = 'faces',
+  SIDECAR = 'sidecar',
+}
+
+export enum WithProperty {
+  SIDECAR = 'sidecar',
 }
 
 export const IAssetRepository = 'IAssetRepository';
@@ -37,6 +42,7 @@ export const IAssetRepository = 'IAssetRepository';
 export interface IAssetRepository {
   getByIds(ids: string[]): Promise<AssetEntity[]>;
   getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
+  getWith(pagination: PaginationOptions, property: WithProperty): Paginated<AssetEntity>;
   getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
   deleteAll(ownerId: string): Promise<void>;
   getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;

server/libs/domain/src/job/job.constants.ts

@@ -8,6 +8,7 @@ export enum QueueName {
   BACKGROUND_TASK = 'background-task-queue',
   STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration-queue',
   SEARCH = 'search-queue',
+  SIDECAR = 'sidecar-queue',
 }
 
 export enum JobCommand {
@@ -72,6 +73,11 @@ export enum JobName {
   // clip
   QUEUE_ENCODE_CLIP = 'queue-clip-encode',
   ENCODE_CLIP = 'clip-encode',
+
+  // XMP sidecars
+  QUEUE_SIDECAR = 'queue-sidecar',
+  SIDECAR_DISCOVERY = 'sidecar-discovery',
+  SIDECAR_SYNC = 'sidecar-sync',
 }
 
 export const JOBS_ASSET_PAGINATION_SIZE = 1000;

server/libs/domain/src/job/job.repository.ts

@@ -50,6 +50,11 @@ export type JobItem =
   | { name: JobName.EXIF_EXTRACTION; data: IAssetJob }
   | { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetJob }
 
+  // Sidecar Scanning
+  | { name: JobName.QUEUE_SIDECAR; data: IBaseJob }
+  | { name: JobName.SIDECAR_DISCOVERY; data: IAssetJob }
+  | { name: JobName.SIDECAR_SYNC; data: IAssetJob }
+
   // Object Tagging
   | { name: JobName.QUEUE_OBJECT_TAGGING; data: IBaseJob }
   | { name: JobName.DETECT_OBJECTS; data: IAssetJob }

server/libs/domain/src/job/job.service.spec.ts

@@ -67,6 +67,7 @@ describe(JobService.name, () => {
         'thumbnail-generation-queue': expectedJobStatus,
         'video-conversion-queue': expectedJobStatus,
         'recognize-faces-queue': expectedJobStatus,
+        'sidecar-queue': expectedJobStatus,
       });
     });
   });

server/libs/domain/src/job/job.service.ts

@@ -76,6 +76,9 @@ export class JobService {
       case QueueName.METADATA_EXTRACTION:
         return this.jobRepository.queue({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force } });
 
+      case QueueName.SIDECAR:
+        return this.jobRepository.queue({ name: JobName.QUEUE_SIDECAR, data: { force } });
+
       case QueueName.THUMBNAIL_GENERATION:
         return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } });
 

server/libs/domain/src/job/response-dto/all-job-status-response.dto.ts

@@ -56,4 +56,7 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
 
   @ApiProperty({ type: JobStatusDto })
   [QueueName.RECOGNIZE_FACES]!: JobStatusDto;
+
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.SIDECAR]!: JobStatusDto;
 }

server/libs/domain/src/storage-template/storage-template.service.ts

@@ -82,14 +82,32 @@ export class StorageTemplateService {
     if (asset.originalPath !== destination) {
       const source = asset.originalPath;
 
+      let sidecarMoved = false;
       try {
         await this.storageRepository.moveFile(asset.originalPath, destination);
+
+        let sidecarDestination;
         try {
-          await this.assetRepository.save({ id: asset.id, originalPath: destination });
+          if (asset.sidecarPath) {
+            sidecarDestination = `${destination}.xmp`;
+            await this.storageRepository.moveFile(asset.sidecarPath, sidecarDestination);
+            sidecarMoved = true;
+          }
+
+          await this.assetRepository.save({ id: asset.id, originalPath: destination, sidecarPath: sidecarDestination });
           asset.originalPath = destination;
+          asset.sidecarPath = sidecarDestination || null;
         } catch (error: any) {
           this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack);
+
+          // Either sidecar move failed or the save failed. Eithr way, move media back
           await this.storageRepository.moveFile(destination, source);
+
+          if (asset.sidecarPath && sidecarDestination && sidecarMoved) {
+            // If the sidecar was moved, that means the saved failed. So move both the sidecar and the
+            // media back into their original positions
+            await this.storageRepository.moveFile(sidecarDestination, asset.sidecarPath);
+          }
         }
       } catch (error: any) {
         this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });

server/libs/domain/test/asset.repository.mock.ts

@@ -4,6 +4,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
   return {
     getByIds: jest.fn(),
     getWithout: jest.fn(),
+    getWith: jest.fn(),
     getFirstAssetForAlbumId: jest.fn(),
     getAll: jest.fn().mockResolvedValue({
       items: [],

server/libs/domain/test/fixtures.ts

@@ -163,6 +163,7 @@ export const assetEntityStub = {
     tags: [],
     sharedLinks: [],
     faces: [],
+    sidecarPath: null,
   }),
   image: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -191,6 +192,7 @@ export const assetEntityStub = {
     sharedLinks: [],
     originalFileName: 'asset-id.ext',
     faces: [],
+    sidecarPath: null,
   }),
   video: Object.freeze<AssetEntity>({
     id: 'asset-id',
@@ -219,6 +221,7 @@ export const assetEntityStub = {
     tags: [],
     sharedLinks: [],
     faces: [],
+    sidecarPath: null,
   }),
   livePhotoMotionAsset: Object.freeze({
     id: 'live-photo-motion-asset',
@@ -252,6 +255,7 @@ export const assetEntityStub = {
     checksum: Buffer.from('file hash', 'utf8'),
     originalPath: '/original/path.ext',
     resizePath: '/uploads/user-id/thumbs/path.ext',
+    sidecarPath: null,
     type: AssetType.IMAGE,
     webpPath: null,
     encodedVideoPath: null,
@@ -719,6 +723,7 @@ export const sharedLinkStub = {
           tags: [],
           sharedLinks: [],
           faces: [],
+          sidecarPath: null,
         },
       ],
     },

server/libs/infra/src/entities/asset.entity.ts

@@ -95,6 +95,9 @@ export class AssetEntity {
   @Column({ type: 'varchar' })
   originalFileName!: string;
 
+  @Column({ type: 'varchar', nullable: true })
+  sidecarPath!: string | null;
+
   @OneToOne(() => ExifEntity, (exifEntity) => exifEntity.asset)
   exifInfo?: ExifEntity;
 

server/libs/infra/src/migrations/1684273840676-AddSidecarFile.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddSidecarFile1684273840676 implements MigrationInterface {
+    name = 'AddSidecarFile1684273840676'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "assets" ADD "sidecarPath" character varying`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "sidecarPath"`);
+    }
+
+}

server/libs/infra/src/repositories/asset.repository.ts

@@ -7,6 +7,7 @@ import {
   Paginated,
   PaginationOptions,
   WithoutProperty,
+  WithProperty,
 } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
@@ -161,6 +162,13 @@ export class AssetRepository implements IAssetRepository {
         };
         break;
 
+      case WithoutProperty.SIDECAR:
+        where = [
+          { sidecarPath: IsNull(), isVisible: true },
+          { sidecarPath: '', isVisible: true },
+        ];
+        break;
+
       default:
         throw new Error(`Invalid getWithout property: ${property}`);
     }
@@ -175,6 +183,27 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  getWith(pagination: PaginationOptions, property: WithProperty): Paginated<AssetEntity> {
+    let where: FindOptionsWhere<AssetEntity> | FindOptionsWhere<AssetEntity>[] = {};
+
+    switch (property) {
+      case WithProperty.SIDECAR:
+        where = [{ sidecarPath: Not(IsNull()), isVisible: true }];
+        break;
+
+      default:
+        throw new Error(`Invalid getWith property: ${property}`);
+    }
+
+    return paginate(this.repository, pagination, {
+      where,
+      order: {
+        // Ensures correct order when paginating
+        createdAt: 'ASC',
+      },
+    });
+  }
+
   getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null> {
     return this.repository.findOne({
       where: { albums: { id: albumId } },

server/libs/infra/src/repositories/job.repository.ts

@@ -15,6 +15,7 @@ export class JobRepository implements IJobRepository {
     [QueueName.VIDEO_CONVERSION]: this.videoTranscode,
     [QueueName.BACKGROUND_TASK]: this.backgroundTask,
     [QueueName.SEARCH]: this.searchIndex,
+    [QueueName.SIDECAR]: this.sidecar,
   };
 
   constructor(
@@ -27,6 +28,7 @@ export class JobRepository implements IJobRepository {
     @InjectQueue(QueueName.THUMBNAIL_GENERATION) private generateThumbnail: Queue,
     @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IAssetJob | IBaseJob>,
     @InjectQueue(QueueName.SEARCH) private searchIndex: Queue,
+    @InjectQueue(QueueName.SIDECAR) private sidecar: Queue<IBaseJob>,
   ) {}
 
   async getQueueStatus(name: QueueName): Promise<QueueStatus> {
@@ -83,6 +85,12 @@ export class JobRepository implements IJobRepository {
         await this.metadataExtraction.add(item.name, item.data);
         break;
 
+      case JobName.QUEUE_SIDECAR:
+      case JobName.SIDECAR_DISCOVERY:
+      case JobName.SIDECAR_SYNC:
+        await this.sidecar.add(item.name, item.data);
+        break;
+
       case JobName.QUEUE_RECOGNIZE_FACES:
       case JobName.RECOGNIZE_FACES:
         await this.recognizeFaces.add(item.name, item.data);