import { IAssetRepository, IBaseJob, ICryptoRepository, IEntityJob, IGeocodingRepository, IJobRepository, IStorageRepository, JobName, JOBS_ASSET_PAGINATION_SIZE, QueueName, StorageCore, StorageFolder, usePagination, WithoutProperty, } from '@app/domain'; import { AssetEntity, AssetType, ExifEntity } from '@app/infra/entities'; import { Inject, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; import tz_lookup from '@photostructure/tz-lookup'; import { exiftool, Tags } from 'exiftool-vendored'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import { Duration } from 'luxon'; import fs from 'node:fs'; import path from 'node:path'; import sharp from 'sharp'; import { Repository } from 'typeorm/repository/Repository'; import { promisify } from 'util'; import { parseLatitude, parseLongitude } from '../utils/exif/coordinates'; import { exifTimeZone, exifToDate } from '../utils/exif/date-time'; import { parseISO } from '../utils/exif/iso'; import { toNumberOrNull } from '../utils/numbers'; const ffprobe = promisify(ffmpeg.ffprobe); interface DirectoryItem { Length?: number; Mime: string; Padding?: number; Semantic?: string; } interface DirectoryEntry { Item: DirectoryItem; } interface ImmichTags extends Tags { ContentIdentifier?: string; MotionPhoto?: number; MotionPhotoVersion?: number; MotionPhotoPresentationTimestampUs?: number; } export class MetadataExtractionProcessor { private logger = new Logger(MetadataExtractionProcessor.name); private reverseGeocodingEnabled: boolean; private storageCore = new StorageCore(); constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository, @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @InjectRepository(ExifEntity) private exifRepository: Repository, configService: ConfigService, ) { this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING'); } async init(deleteCache = false) { this.logger.log(`Reverse geocoding is ${this.reverseGeocodingEnabled ? 'enabled' : 'disabled'}`); if (!this.reverseGeocodingEnabled) { return; } try { if (deleteCache) { await this.geocodingRepository.deleteCache(); } this.logger.log('Initializing Reverse Geocoding'); await this.jobRepository.pause(QueueName.METADATA_EXTRACTION); await this.geocodingRepository.init(); await this.jobRepository.resume(QueueName.METADATA_EXTRACTION); this.logger.log('Reverse Geocoding Initialized'); } catch (error: any) { this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack); } } async handleQueueMetadataExtraction(job: IBaseJob) { const { force } = job; const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { return force ? this.assetRepository.getAll(pagination) : this.assetRepository.getWithout(pagination, WithoutProperty.EXIF); }); for await (const assets of assetPagination) { for (const asset of assets) { await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } }); } } return true; } async handleMetadataExtraction({ id }: IEntityJob) { const [asset] = await this.assetRepository.getByIds([id]); if (!asset || !asset.isVisible) { return false; } if (asset.type === AssetType.VIDEO) { return this.handleVideoMetadataExtraction(asset); } else { return this.handlePhotoMetadataExtraction(asset); } } async addExtractedLivePhoto(sourceAsset: AssetEntity, video: string, created: Date | null): Promise { if (sourceAsset.livePhotoVideoId) { const [liveAsset] = await this.assetRepository.getByIds([sourceAsset.livePhotoVideoId]); // already exists so no need to generate ID. if (liveAsset.originalPath == video) { return liveAsset; } liveAsset.originalPath = video; return this.assetRepository.save(liveAsset); } const liveAsset = await this.assetRepository.save({ ownerId: sourceAsset.ownerId, owner: sourceAsset.owner, checksum: await this.cryptoRepository.hashFile(video), originalPath: video, fileCreatedAt: created ?? sourceAsset.fileCreatedAt, fileModifiedAt: sourceAsset.fileModifiedAt, deviceAssetId: 'NONE', deviceId: 'NONE', type: AssetType.VIDEO, isFavorite: false, isArchived: sourceAsset.isArchived, duration: null, isVisible: false, livePhotoVideo: null, resizePath: null, webpPath: null, thumbhash: null, encodedVideoPath: null, tags: [], sharedLinks: [], originalFileName: path.parse(video).name, faces: [], sidecarPath: null, isReadOnly: sourceAsset.isReadOnly, }); sourceAsset.livePhotoVideoId = liveAsset.id; await this.assetRepository.save(sourceAsset); return liveAsset; } private async extractNewPixelLivePhoto( asset: AssetEntity, directory: DirectoryEntry[], fileCreatedAt: Date | null, ): Promise { if (asset.livePhotoVideoId) { // Already extracted, don't try again. const [ret] = await this.assetRepository.getByIds([asset.livePhotoVideoId]); this.logger.log(`Already extracted asset ${ret.originalPath}.`); return ret; } let foundMotionPhoto = false; let motionPhotoOffsetFromEnd = 0; let motionPhotoLength = 0; // Look for the directory entry with semantic label "MotionPhoto", which is the embedded video. // Then, determine the length from the end of the file to the start of the embedded video. for (const entry of directory) { if (entry.Item.Semantic == 'MotionPhoto') { if (foundMotionPhoto) { this.logger.error(`Asset ${asset.originalPath} has more than one motion photo.`); continue; } foundMotionPhoto = true; motionPhotoLength = entry.Item.Length ?? 0; } if (foundMotionPhoto) { motionPhotoOffsetFromEnd += entry.Item.Length ?? 0; motionPhotoOffsetFromEnd += entry.Item.Padding ?? 0; } } if (!foundMotionPhoto || motionPhotoLength == 0) { return null; } return this.extractEmbeddedVideo(asset, motionPhotoOffsetFromEnd, motionPhotoLength, fileCreatedAt); } private async extractEmbeddedVideo( asset: AssetEntity, offsetFromEnd: number, length: number | null, fileCreatedAt: Date | null, ) { let file = null; try { file = await fs.promises.open(asset.originalPath); let extracted = null; // Read in embedded video. const stat = await file.stat(); if (length == null) { length = offsetFromEnd; } const offset = stat.size - offsetFromEnd; extracted = await file.read({ buffer: Buffer.alloc(length), position: offset, length: length, }); // Write out extracted video, and add it to the asset repository. const encodedVideoFolder = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId); this.storageRepository.mkdirSync(encodedVideoFolder); const livePhotoPath = path.join(encodedVideoFolder, path.parse(asset.originalPath).name + '.mp4'); await fs.promises.writeFile(livePhotoPath, extracted.buffer); const result = await this.addExtractedLivePhoto(asset, livePhotoPath, fileCreatedAt); await this.handleMetadataExtraction({ id: result.id }); return result; } catch (e) { this.logger.error(`Failed to extract live photo ${asset.originalPath}: ${e}`); return null; } finally { if (file) { await file.close(); } } } private async handlePhotoMetadataExtraction(asset: AssetEntity) { const mediaExifData = await exiftool.read(asset.originalPath).catch((error: any) => { this.logger.warn( `The exifData parsing failed due to ${error} for asset ${asset.id} at ${asset.originalPath}`, error?.stack, ); return null; }); const sidecarExifData = asset.sidecarPath ? await exiftool.read(asset.sidecarPath).catch((error: any) => { this.logger.warn( `The exifData parsing failed due to ${error} for asset ${asset.id} at ${asset.originalPath}`, error?.stack, ); return null; }) : {}; const getExifProperty = ( ...properties: T[] ): NonNullable | string | null => { for (const property of properties) { const value = sidecarExifData?.[property] ?? mediaExifData?.[property]; if (value !== null && value !== undefined) { // Can also be string when the value cannot be parsed return value; } } return null; }; const timeZone = exifTimeZone(getExifProperty('DateTimeOriginal', 'CreateDate') ?? asset.fileCreatedAt); const fileCreatedAt = exifToDate(getExifProperty('DateTimeOriginal', 'CreateDate') ?? asset.fileCreatedAt); const fileModifiedAt = exifToDate(getExifProperty('ModifyDate') ?? asset.fileModifiedAt); const fileStats = fs.statSync(asset.originalPath); const fileSizeInBytes = fileStats.size; const newExif = new ExifEntity(); newExif.assetId = asset.id; newExif.fileSizeInByte = fileSizeInBytes; newExif.make = getExifProperty('Make'); newExif.model = getExifProperty('Model'); newExif.exifImageHeight = toNumberOrNull(getExifProperty('ExifImageHeight', 'ImageHeight')); newExif.exifImageWidth = toNumberOrNull(getExifProperty('ExifImageWidth', 'ImageWidth')); newExif.exposureTime = getExifProperty('ExposureTime'); newExif.orientation = getExifProperty('Orientation')?.toString() ?? null; newExif.dateTimeOriginal = fileCreatedAt; newExif.modifyDate = fileModifiedAt; newExif.timeZone = timeZone; newExif.lensModel = getExifProperty('LensModel'); newExif.fNumber = toNumberOrNull(getExifProperty('FNumber')); newExif.focalLength = toNumberOrNull(getExifProperty('FocalLength')); // Handle array values by converting to string const iso = getExifProperty('ISO')?.toString(); newExif.iso = iso ? parseISO(iso) : null; const latitude = getExifProperty('GPSLatitude'); const longitude = getExifProperty('GPSLongitude'); newExif.latitude = latitude !== null ? parseLatitude(latitude) : null; newExif.longitude = longitude !== null ? parseLongitude(longitude) : null; if (getExifProperty('MotionPhoto')) { // Seen on more recent Pixel phones: starting as early as Pixel 4a, possibly earlier. const rawDirectory = getExifProperty('Directory'); if (Array.isArray(rawDirectory)) { // exiftool-vendor thinks directory is a string, but actually it's an array of DirectoryEntry. const directory = rawDirectory as DirectoryEntry[]; await this.extractNewPixelLivePhoto(asset, directory, fileCreatedAt); } else { this.logger.warn(`Failed to get Pixel motionPhoto information: directory: ${JSON.stringify(rawDirectory)}`); } } else if (getExifProperty('MicroVideo')) { // Seen on earlier Pixel phones - Pixel 2 and earlier, possibly Pixel 3. let offset = getExifProperty('MicroVideoOffset'); // offset from end of file. if (typeof offset == 'string') { offset = parseInt(offset); } if (Number.isNaN(offset) || offset == null) { this.logger.warn( `Failed to get MicroVideo information for ${asset.originalPath}, offset=${getExifProperty( 'MicroVideoOffset', )}`, ); } else { await this.extractEmbeddedVideo(asset, offset, null, fileCreatedAt); } } const projectionType = getExifProperty('ProjectionType'); if (projectionType) { newExif.projectionType = String(projectionType).toUpperCase(); } newExif.livePhotoCID = getExifProperty('MediaGroupUUID'); if (newExif.livePhotoCID && !asset.livePhotoVideoId) { const motionAsset = await this.assetRepository.findLivePhotoMatch({ livePhotoCID: newExif.livePhotoCID, otherAssetId: asset.id, ownerId: asset.ownerId, type: AssetType.VIDEO, }); if (motionAsset) { await this.assetRepository.save({ id: asset.id, livePhotoVideoId: motionAsset.id }); await this.assetRepository.save({ id: motionAsset.id, isVisible: false }); } } await this.applyReverseGeocoding(asset, newExif); /** * IF the EXIF doesn't contain the width and height of the image, * We will use Sharpjs to get the information. */ if (!newExif.exifImageHeight || !newExif.exifImageWidth || !newExif.orientation) { const metadata = await sharp(asset.originalPath).metadata(); if (newExif.exifImageHeight === null) { newExif.exifImageHeight = metadata.height || null; } if (newExif.exifImageWidth === null) { newExif.exifImageWidth = metadata.width || null; } if (newExif.orientation === null) { newExif.orientation = metadata.orientation !== undefined ? `${metadata.orientation}` : null; } } await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); await this.assetRepository.save({ id: asset.id, fileCreatedAt: fileCreatedAt || undefined }); return true; } private async handleVideoMetadataExtraction(asset: AssetEntity) { const data = await ffprobe(asset.originalPath); const durationString = this.extractDuration(data.format.duration || asset.duration); let fileCreatedAt = asset.fileCreatedAt; const videoTags = data.format.tags; if (videoTags) { if (videoTags['com.apple.quicktime.creationdate']) { fileCreatedAt = new Date(videoTags['com.apple.quicktime.creationdate']); } else if (videoTags['creation_time']) { fileCreatedAt = new Date(videoTags['creation_time']); } } const exifData = await exiftool.read(asset.sidecarPath || asset.originalPath).catch((error: any) => { this.logger.warn( `The exifData parsing failed due to ${error} for asset ${asset.id} at ${asset.originalPath}`, error?.stack, ); return null; }); const newExif = new ExifEntity(); newExif.assetId = asset.id; newExif.fileSizeInByte = data.format.size || null; newExif.dateTimeOriginal = fileCreatedAt ? new Date(fileCreatedAt) : null; newExif.modifyDate = null; newExif.timeZone = null; newExif.latitude = null; newExif.longitude = null; newExif.city = null; newExif.state = null; newExif.country = null; newExif.fps = null; newExif.livePhotoCID = exifData?.ContentIdentifier || null; if (newExif.livePhotoCID) { const photoAsset = await this.assetRepository.findLivePhotoMatch({ livePhotoCID: newExif.livePhotoCID, ownerId: asset.ownerId, otherAssetId: asset.id, type: AssetType.IMAGE, }); if (photoAsset) { await this.assetRepository.save({ id: photoAsset.id, livePhotoVideoId: asset.id }); await this.assetRepository.save({ id: asset.id, isVisible: false }); } } if (videoTags && videoTags['location']) { const location = videoTags['location'] as string; const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/; const match = location.match(locationRegex); if (match?.length === 3) { newExif.latitude = parseLatitude(match[1]); newExif.longitude = parseLongitude(match[2]); } } else if (videoTags && videoTags['com.apple.quicktime.location.ISO6709']) { const location = videoTags['com.apple.quicktime.location.ISO6709'] as string; const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/; const match = location.match(locationRegex); if (match?.length === 4) { newExif.latitude = parseLatitude(match[1]); newExif.longitude = parseLongitude(match[2]); } } if (newExif.longitude && newExif.latitude) { try { newExif.timeZone = tz_lookup(newExif.latitude, newExif.longitude); } catch (error: any) { this.logger.warn(`Error while calculating timezone from gps coordinates: ${error}`, error?.stack); } } await this.applyReverseGeocoding(asset, newExif); for (const stream of data.streams) { if (stream.codec_type === 'video') { newExif.exifImageWidth = stream.width || null; newExif.exifImageHeight = stream.height || null; if (typeof stream.rotation === 'string') { newExif.orientation = stream.rotation; } else if (typeof stream.rotation === 'number') { newExif.orientation = `${stream.rotation}`; } else { newExif.orientation = null; } if (stream.r_frame_rate) { const fpsParts = stream.r_frame_rate.split('/'); if (fpsParts.length === 2) { newExif.fps = Math.round(parseInt(fpsParts[0]) / parseInt(fpsParts[1])); } } } } await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); await this.assetRepository.save({ id: asset.id, duration: durationString, fileCreatedAt }); return true; } private async applyReverseGeocoding(asset: AssetEntity, newExif: ExifEntity) { const { latitude, longitude } = newExif; if (this.reverseGeocodingEnabled && longitude && latitude) { try { const { country, state, city } = await this.geocodingRepository.reverseGeocode({ latitude, longitude }); newExif.country = country; newExif.state = state; newExif.city = city; } catch (error: any) { this.logger.warn( `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`, error?.stack, ); } } } private extractDuration(duration: number | string | null) { const videoDurationInSecond = Number(duration); if (!videoDurationInSecond) { return null; } return Duration.fromObject({ seconds: videoDurationInSecond }).toFormat('hh:mm:ss.SSS'); } }