refactor(server): Move metadata extraction to domain (#4243)

* use storageRepository in metadata extraction

* move metadata extraction processor to domain

* cleanup infra/domain

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/metadata/geocoding.repository.ts

@@ -1,20 +0,0 @@
-import { InitOptions } from 'local-reverse-geocoder';
-
-export const IGeocodingRepository = 'IGeocodingRepository';
-
-export interface GeoPoint {
-  latitude: number;
-  longitude: number;
-}
-
-export interface ReverseGeocodeResult {
-  country: string | null;
-  state: string | null;
-  city: string | null;
-}
-
-export interface IGeocodingRepository {
-  init(options: Partial<InitOptions>): Promise<void>;
-  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
-  deleteCache(): Promise<void>;
-}

server/src/domain/metadata/index.ts

@@ -1,2 +1,2 @@
-export * from './geocoding.repository';
+export * from './metadata.repository';
 export * from './metadata.service';

server/src/domain/metadata/metadata.repository.ts

@@ -0,0 +1,31 @@
+import { Tags } from 'exiftool-vendored';
+import { InitOptions } from 'local-reverse-geocoder';
+
+export const IMetadataRepository = 'IMetadataRepository';
+
+export interface GeoPoint {
+  latitude: number;
+  longitude: number;
+}
+
+export interface ReverseGeocodeResult {
+  country: string | null;
+  state: string | null;
+  city: string | null;
+}
+
+export interface ImmichTags extends Tags {
+  ContentIdentifier?: string;
+  MotionPhoto?: number;
+  MotionPhotoVersion?: number;
+  MotionPhotoPresentationTimestampUs?: number;
+  MediaGroupUUID?: string;
+  ImagePixelDepth?: string;
+}
+
+export interface IMetadataRepository {
+  init(options: Partial<InitOptions>): Promise<void>;
+  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
+  deleteCache(): Promise<void>;
+  getExifTags(path: string): Promise<ImmichTags | null>;
+}

server/src/domain/metadata/metadata.service.spec.ts

@@ -1,22 +1,43 @@
-import { assetStub, newAssetRepositoryMock, newJobRepositoryMock, newStorageRepositoryMock } from '@test';
+import {
+  assetStub,
+  newAlbumRepositoryMock,
+  newAssetRepositoryMock,
+  newCryptoRepositoryMock,
+  newJobRepositoryMock,
+  newMetadataRepositoryMock,
+  newStorageRepositoryMock,
+  newSystemConfigRepositoryMock,
+} from '@test';
 import { constants } from 'fs/promises';
+import { IAlbumRepository } from '../album';
 import { IAssetRepository, WithProperty, WithoutProperty } from '../asset';
+import { ICryptoRepository } from '../crypto';
 import { IJobRepository, JobName } from '../job';
 import { IStorageRepository } from '../storage';
+import { ISystemConfigRepository } from '../system-config';
+import { IMetadataRepository } from './metadata.repository';
 import { MetadataService } from './metadata.service';
 
 describe(MetadataService.name, () => {
-  let sut: MetadataService;
+  let albumMock: jest.Mocked<IAlbumRepository>;
   let assetMock: jest.Mocked<IAssetRepository>;
+  let configMock: jest.Mocked<ISystemConfigRepository>;
+  let cryptoRepository: jest.Mocked<ICryptoRepository>;
   let jobMock: jest.Mocked<IJobRepository>;
+  let metadataMock: jest.Mocked<IMetadataRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
+  let sut: MetadataService;
 
   beforeEach(async () => {
+    albumMock = newAlbumRepositoryMock();
     assetMock = newAssetRepositoryMock();
+    configMock = newSystemConfigRepositoryMock();
+    cryptoRepository = newCryptoRepositoryMock();
     jobMock = newJobRepositoryMock();
+    metadataMock = newMetadataRepositoryMock();
     storageMock = newStorageRepositoryMock();
 
-    sut = new MetadataService(assetMock, jobMock, storageMock);
+    sut = new MetadataService(albumMock, assetMock, cryptoRepository, jobMock, metadataMock, storageMock, configMock);
   });
 
   it('should be defined', () => {

server/src/domain/metadata/metadata.service.ts

@@ -1,16 +1,148 @@
-import { Inject } from '@nestjs/common';
+import { AssetEntity, AssetType, ExifEntity } from '@app/infra/entities';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { ExifDateTime } from 'exiftool-vendored';
+import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime';
 import { constants } from 'fs/promises';
-import { IAssetRepository, WithoutProperty, WithProperty } from '../asset';
+import { Duration } from 'luxon';
+import { IAlbumRepository } from '../album';
+import { IAssetRepository, WithProperty, WithoutProperty } from '../asset';
+import { ICryptoRepository } from '../crypto';
 import { usePagination } from '../domain.util';
-import { IBaseJob, IEntityJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
-import { IStorageRepository } from '../storage';
+import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
+import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
+import { FeatureFlag, ISystemConfigRepository, SystemConfigCore } from '../system-config';
+import { IMetadataRepository, ImmichTags } from './metadata.repository';
 
+interface DirectoryItem {
+  Length?: number;
+  Mime: string;
+  Padding?: number;
+  Semantic?: string;
+}
+
+interface DirectoryEntry {
+  Item: DirectoryItem;
+}
+
+const exifDate = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.toDate() : null);
+// exiftool returns strings when it fails to parse non-string values, so this is used where a string is not expected
+const validate = <T>(value: T): T | null => (typeof value === 'string' ? null : value ?? null);
+
+@Injectable()
 export class MetadataService {
+  private logger = new Logger(MetadataService.name);
+  private storageCore: StorageCore;
+  private configCore: SystemConfigCore;
+  private oldCities?: string;
+
   constructor(
+    @Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
+    @Inject(IMetadataRepository) private repository: IMetadataRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
-  ) {}
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
+  ) {
+    this.storageCore = new StorageCore(storageRepository);
+    this.configCore = new SystemConfigCore(configRepository);
+    this.configCore.config$.subscribe(() => this.init());
+  }
+
+  async init(deleteCache = false) {
+    const { reverseGeocoding } = await this.configCore.getConfig();
+    const { citiesFileOverride } = reverseGeocoding;
+
+    if (!reverseGeocoding.enabled) {
+      return;
+    }
+
+    try {
+      if (deleteCache) {
+        await this.repository.deleteCache();
+      } else if (this.oldCities && this.oldCities === citiesFileOverride) {
+        return;
+      }
+
+      await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
+      await this.repository.init({ citiesFileOverride });
+      await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
+
+      this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`);
+      this.oldCities = citiesFileOverride;
+    } catch (error: Error | any) {
+      this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
+    }
+  }
+
+  async handleLivePhotoLinking(job: IEntityJob) {
+    const { id } = job;
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset?.exifInfo) {
+      return false;
+    }
+
+    if (!asset.exifInfo.livePhotoCID) {
+      return true;
+    }
+
+    const otherType = asset.type === AssetType.VIDEO ? AssetType.IMAGE : AssetType.VIDEO;
+    const match = await this.assetRepository.findLivePhotoMatch({
+      livePhotoCID: asset.exifInfo.livePhotoCID,
+      ownerId: asset.ownerId,
+      otherAssetId: asset.id,
+      type: otherType,
+    });
+
+    if (!match) {
+      return true;
+    }
+
+    const [photoAsset, motionAsset] = asset.type === AssetType.IMAGE ? [asset, match] : [match, asset];
+
+    await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: motionAsset.id });
+    await this.assetRepository.save({ id: motionAsset.id, isVisible: false });
+    await this.albumRepository.removeAsset(motionAsset.id);
+
+    return true;
+  }
+
+  async handleQueueMetadataExtraction(job: IBaseJob) {
+    const { force } = job;
+    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
+      return force
+        ? this.assetRepository.getAll(pagination)
+        : this.assetRepository.getWithout(pagination, WithoutProperty.EXIF);
+    });
+
+    for await (const assets of assetPagination) {
+      for (const asset of assets) {
+        await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } });
+      }
+    }
+
+    return true;
+  }
+
+  async handleMetadataExtraction({ id }: IEntityJob) {
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset || !asset.isVisible) {
+      return false;
+    }
+
+    const { exifData, tags } = await this.exifData(asset);
+
+    await this.applyMotionPhotos(asset, tags);
+    await this.applyReverseGeocoding(asset, exifData);
+    await this.assetRepository.upsertExif(exifData);
+    await this.assetRepository.save({
+      id: asset.id,
+      duration: tags.Duration ? this.getDuration(tags.Duration) : null,
+      fileCreatedAt: exifData.dateTimeOriginal ?? undefined,
+    });
+
+    return true;
+  }
 
   async handleQueueSidecar(job: IBaseJob) {
     const { force } = job;
@@ -51,4 +183,156 @@ export class MetadataService {
 
     return true;
   }
+
+  private async applyReverseGeocoding(asset: AssetEntity, exifData: ExifEntity) {
+    const { latitude, longitude } = exifData;
+    if (!(await this.configCore.hasFeature(FeatureFlag.REVERSE_GEOCODING)) || !longitude || !latitude) {
+      return;
+    }
+
+    try {
+      const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude });
+      Object.assign(exifData, { city, state, country });
+    } catch (error: Error | any) {
+      this.logger.warn(
+        `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
+        error?.stack,
+      );
+    }
+  }
+
+  private async applyMotionPhotos(asset: AssetEntity, tags: ImmichTags) {
+    if (asset.type !== AssetType.IMAGE || asset.livePhotoVideoId) {
+      return;
+    }
+
+    const rawDirectory = tags.Directory;
+    const isMotionPhoto = tags.MotionPhoto;
+    const isMicroVideo = tags.MicroVideo;
+    const videoOffset = tags.MicroVideoOffset;
+    const directory = Array.isArray(rawDirectory) ? (rawDirectory as DirectoryEntry[]) : null;
+
+    let length = 0;
+    let padding = 0;
+
+    if (isMotionPhoto && directory) {
+      for (const entry of directory) {
+        if (entry.Item.Semantic == 'MotionPhoto') {
+          length = entry.Item.Length ?? 0;
+          padding = entry.Item.Padding ?? 0;
+          break;
+        }
+      }
+    }
+
+    if (isMicroVideo && typeof videoOffset === 'number') {
+      length = videoOffset;
+    }
+
+    if (!length) {
+      return;
+    }
+
+    this.logger.debug(`Starting motion photo video extraction (${asset.id})`);
+
+    try {
+      const stat = await this.storageRepository.stat(asset.originalPath);
+      const position = stat.size - length - padding;
+      const video = await this.storageRepository.readFile(asset.originalPath, {
+        buffer: Buffer.alloc(length),
+        position,
+        length,
+      });
+      const checksum = await this.cryptoRepository.hashSha1(video);
+
+      let motionAsset = await this.assetRepository.getByChecksum(asset.ownerId, checksum);
+      if (!motionAsset) {
+        motionAsset = await this.assetRepository.save({
+          libraryId: asset.libraryId,
+          type: AssetType.VIDEO,
+          fileCreatedAt: asset.fileCreatedAt ?? asset.createdAt,
+          fileModifiedAt: asset.fileModifiedAt,
+          checksum,
+          ownerId: asset.ownerId,
+          originalPath: this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`),
+          originalFileName: asset.originalFileName,
+          isVisible: false,
+          isReadOnly: true,
+          deviceAssetId: 'NONE',
+          deviceId: 'NONE',
+        });
+
+        await this.storageRepository.writeFile(asset.originalPath, video);
+
+        await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: motionAsset.id } });
+      }
+
+      await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id });
+
+      this.logger.debug(`Finished motion photo video extraction (${asset.id})`);
+    } catch (error: Error | any) {
+      this.logger.error(`Failed to extract live photo ${asset.originalPath}: ${error}`, error?.stack);
+    }
+  }
+
+  private async exifData(asset: AssetEntity): Promise<{ exifData: ExifEntity; tags: ImmichTags }> {
+    const stats = await this.storageRepository.stat(asset.originalPath);
+    const mediaTags = await this.repository.getExifTags(asset.originalPath);
+    const sidecarTags = asset.sidecarPath ? await this.repository.getExifTags(asset.sidecarPath) : null;
+    const tags = { ...mediaTags, ...sidecarTags };
+
+    this.logger.verbose('Exif Tags', tags);
+
+    return {
+      exifData: <ExifEntity>{
+        // altitude: tags.GPSAltitude ?? null,
+        assetId: asset.id,
+        bitsPerSample: this.getBitsPerSample(tags),
+        colorspace: tags.ColorSpace ?? null,
+        dateTimeOriginal: exifDate(firstDateTime(tags)) ?? asset.fileCreatedAt,
+        exifImageHeight: validate(tags.ImageHeight),
+        exifImageWidth: validate(tags.ImageWidth),
+        exposureTime: tags.ExposureTime ?? null,
+        fileSizeInByte: stats.size,
+        fNumber: validate(tags.FNumber),
+        focalLength: validate(tags.FocalLength),
+        fps: validate(tags.VideoFrameRate),
+        iso: validate(tags.ISO),
+        latitude: validate(tags.GPSLatitude),
+        lensModel: tags.LensModel ?? null,
+        livePhotoCID: (asset.type === AssetType.VIDEO ? tags.ContentIdentifier : tags.MediaGroupUUID) ?? null,
+        longitude: validate(tags.GPSLongitude),
+        make: tags.Make ?? null,
+        model: tags.Model ?? null,
+        modifyDate: exifDate(tags.ModifyDate) ?? asset.fileModifiedAt,
+        orientation: validate(tags.Orientation)?.toString() ?? null,
+        profileDescription: tags.ProfileDescription || tags.ProfileName || null,
+        projectionType: tags.ProjectionType ? String(tags.ProjectionType).toUpperCase() : null,
+        timeZone: tags.tz,
+      },
+      tags,
+    };
+  }
+
+  private getBitsPerSample(tags: ImmichTags): number | null {
+    const bitDepthTags = [
+      tags.BitsPerSample,
+      tags.ComponentBitDepth,
+      tags.ImagePixelDepth,
+      tags.BitDepth,
+      tags.ColorBitDepth,
+      // `numericTags` doesn't parse values like '12 12 12'
+    ].map((tag) => (typeof tag === 'string' ? Number.parseInt(tag) : tag));
+
+    let bitsPerSample = bitDepthTags.find((tag) => typeof tag === 'number' && !Number.isNaN(tag)) ?? null;
+    if (bitsPerSample && bitsPerSample >= 24 && bitsPerSample % 3 === 0) {
+      bitsPerSample /= 3; // converts per-pixel bit depth to per-channel
+    }
+
+    return bitsPerSample;
+  }
+
+  private getDuration(seconds?: number): string {
+    return Duration.fromObject({ seconds }).toFormat('hh:mm:ss.SSS');
+  }
 }

server/src/domain/storage/storage.repository.ts

@@ -1,4 +1,5 @@
 import { Stats } from 'fs';
+import { FileReadOptions } from 'fs/promises';
 import { Readable } from 'stream';
 import { CrawlOptionsDto } from '../library';
 
@@ -24,6 +25,8 @@ export const IStorageRepository = 'IStorageRepository';
 export interface IStorageRepository {
   createZipStream(): ImmichZipStream;
   createReadStream(filepath: string, mimeType?: string | null): Promise<ImmichReadStream>;
+  readFile(filepath: string, options?: FileReadOptions<Buffer>): Promise<Buffer>;
+  writeFile(filepath: string, buffer: Buffer): Promise<void>;
   unlink(filepath: string): Promise<void>;
   unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
   removeEmptyDirs(folder: string, self?: boolean): Promise<void>;