mirror of
				https://github.com/KevinMidboe/immich.git
				synced 2025-10-29 17:40:28 +00:00 
			
		
		
		
	feat(web/server): Add options to rerun job on all assets (#1422)
This commit is contained in:
		
							
								
								
									
										1
									
								
								mobile/openapi/doc/JobCommandDto.md
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										1
									
								
								mobile/openapi/doc/JobCommandDto.md
									
									
									
										generated
									
									
									
								
							| @@ -9,6 +9,7 @@ import 'package:openapi/api.dart'; | |||||||
| Name | Type | Description | Notes | Name | Type | Description | Notes | ||||||
| ------------ | ------------- | ------------- | ------------- | ------------ | ------------- | ------------- | ------------- | ||||||
| **command** | [**JobCommand**](JobCommand.md) |  |  | **command** | [**JobCommand**](JobCommand.md) |  |  | ||||||
|  | **includeAllAssets** | **bool** |  |  | ||||||
| 
 | 
 | ||||||
| [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) | ||||||
| 
 | 
 | ||||||
|   | |||||||
							
								
								
									
										14
									
								
								mobile/openapi/lib/model/job_command_dto.dart
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										14
									
								
								mobile/openapi/lib/model/job_command_dto.dart
									
									
									
										generated
									
									
									
								
							| @@ -14,25 +14,31 @@ class JobCommandDto { | |||||||
|   /// Returns a new [JobCommandDto] instance. |   /// Returns a new [JobCommandDto] instance. | ||||||
|   JobCommandDto({ |   JobCommandDto({ | ||||||
|     required this.command, |     required this.command, | ||||||
|  |     required this.includeAllAssets, | ||||||
|   }); |   }); | ||||||
| 
 | 
 | ||||||
|   JobCommand command; |   JobCommand command; | ||||||
| 
 | 
 | ||||||
|  |   bool includeAllAssets; | ||||||
|  | 
 | ||||||
|   @override |   @override | ||||||
|   bool operator ==(Object other) => identical(this, other) || other is JobCommandDto && |   bool operator ==(Object other) => identical(this, other) || other is JobCommandDto && | ||||||
|      other.command == command; |      other.command == command && | ||||||
|  |      other.includeAllAssets == includeAllAssets; | ||||||
| 
 | 
 | ||||||
|   @override |   @override | ||||||
|   int get hashCode => |   int get hashCode => | ||||||
|     // ignore: unnecessary_parenthesis |     // ignore: unnecessary_parenthesis | ||||||
|     (command.hashCode); |     (command.hashCode) + | ||||||
|  |     (includeAllAssets.hashCode); | ||||||
| 
 | 
 | ||||||
|   @override |   @override | ||||||
|   String toString() => 'JobCommandDto[command=$command]'; |   String toString() => 'JobCommandDto[command=$command, includeAllAssets=$includeAllAssets]'; | ||||||
| 
 | 
 | ||||||
|   Map<String, dynamic> toJson() { |   Map<String, dynamic> toJson() { | ||||||
|     final json = <String, dynamic>{}; |     final json = <String, dynamic>{}; | ||||||
|       json[r'command'] = this.command; |       json[r'command'] = this.command; | ||||||
|  |       json[r'includeAllAssets'] = this.includeAllAssets; | ||||||
|     return json; |     return json; | ||||||
|   } |   } | ||||||
| 
 | 
 | ||||||
| @@ -56,6 +62,7 @@ class JobCommandDto { | |||||||
| 
 | 
 | ||||||
|       return JobCommandDto( |       return JobCommandDto( | ||||||
|         command: JobCommand.fromJson(json[r'command'])!, |         command: JobCommand.fromJson(json[r'command'])!, | ||||||
|  |         includeAllAssets: mapValueOfType<bool>(json, r'includeAllAssets')!, | ||||||
|       ); |       ); | ||||||
|     } |     } | ||||||
|     return null; |     return null; | ||||||
| @@ -106,6 +113,7 @@ class JobCommandDto { | |||||||
|   /// The list of required keys that must be present in a JSON. |   /// The list of required keys that must be present in a JSON. | ||||||
|   static const requiredKeys = <String>{ |   static const requiredKeys = <String>{ | ||||||
|     'command', |     'command', | ||||||
|  |     'includeAllAssets', | ||||||
|   }; |   }; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|   | |||||||
							
								
								
									
										5
									
								
								mobile/openapi/test/job_command_dto_test.dart
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										5
									
								
								mobile/openapi/test/job_command_dto_test.dart
									
									
									
										generated
									
									
									
								
							| @@ -21,6 +21,11 @@ void main() { | |||||||
|       // TODO |       // TODO | ||||||
|     }); |     }); | ||||||
| 
 | 
 | ||||||
|  |     // bool includeAllAssets | ||||||
|  |     test('to test the property `includeAllAssets`', () async { | ||||||
|  |       // TODO | ||||||
|  |     }); | ||||||
|  | 
 | ||||||
| 
 | 
 | ||||||
|   }); |   }); | ||||||
| 
 | 
 | ||||||
|   | |||||||
| @@ -29,6 +29,8 @@ export interface IAssetRepository { | |||||||
|     livePhotoAssetEntity?: AssetEntity, |     livePhotoAssetEntity?: AssetEntity, | ||||||
|   ): Promise<AssetEntity>; |   ): Promise<AssetEntity>; | ||||||
|   update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>; |   update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>; | ||||||
|  |   getAll(): Promise<AssetEntity[]>; | ||||||
|  |   getAllVideos(): Promise<AssetEntity[]>; | ||||||
|   getAllByUserId(userId: string, dto: AssetSearchDto): Promise<AssetEntity[]>; |   getAllByUserId(userId: string, dto: AssetSearchDto): Promise<AssetEntity[]>; | ||||||
|   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>; |   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>; | ||||||
|   getById(assetId: string): Promise<AssetEntity>; |   getById(assetId: string): Promise<AssetEntity>; | ||||||
| @@ -61,6 +63,22 @@ export class AssetRepository implements IAssetRepository { | |||||||
|     @Inject(ITagRepository) private _tagRepository: ITagRepository, |     @Inject(ITagRepository) private _tagRepository: ITagRepository, | ||||||
|   ) {} |   ) {} | ||||||
|  |  | ||||||
|  |   async getAllVideos(): Promise<AssetEntity[]> { | ||||||
|  |     return await this.assetRepository.find({ | ||||||
|  |       where: { type: AssetType.VIDEO }, | ||||||
|  |     }); | ||||||
|  |   } | ||||||
|  |  | ||||||
|  |   async getAll(): Promise<AssetEntity[]> { | ||||||
|  |     return await this.assetRepository.find({ | ||||||
|  |       where: { isVisible: true }, | ||||||
|  |       relations: { | ||||||
|  |         exifInfo: true, | ||||||
|  |         smartInfo: true, | ||||||
|  |       }, | ||||||
|  |     }); | ||||||
|  |   } | ||||||
|  |  | ||||||
|   async getAssetWithNoSmartInfo(): Promise<AssetEntity[]> { |   async getAssetWithNoSmartInfo(): Promise<AssetEntity[]> { | ||||||
|     return await this.assetRepository |     return await this.assetRepository | ||||||
|       .createQueryBuilder('asset') |       .createQueryBuilder('asset') | ||||||
|   | |||||||
| @@ -123,6 +123,8 @@ describe('AssetService', () => { | |||||||
|     assetRepositoryMock = { |     assetRepositoryMock = { | ||||||
|       create: jest.fn(), |       create: jest.fn(), | ||||||
|       update: jest.fn(), |       update: jest.fn(), | ||||||
|  |       getAll: jest.fn(), | ||||||
|  |       getAllVideos: jest.fn(), | ||||||
|       getAllByUserId: jest.fn(), |       getAllByUserId: jest.fn(), | ||||||
|       getAllByDeviceId: jest.fn(), |       getAllByDeviceId: jest.fn(), | ||||||
|       getAssetCountByTimeBucket: jest.fn(), |       getAssetCountByTimeBucket: jest.fn(), | ||||||
|   | |||||||
| @@ -1,5 +1,5 @@ | |||||||
| import { ApiProperty } from '@nestjs/swagger'; | import { ApiProperty } from '@nestjs/swagger'; | ||||||
| import { IsIn, IsNotEmpty } from 'class-validator'; | import { IsBoolean, IsIn, IsNotEmpty, IsOptional } from 'class-validator'; | ||||||
|  |  | ||||||
| export class JobCommandDto { | export class JobCommandDto { | ||||||
|   @IsNotEmpty() |   @IsNotEmpty() | ||||||
| @@ -9,4 +9,8 @@ export class JobCommandDto { | |||||||
|     enumName: 'JobCommand', |     enumName: 'JobCommand', | ||||||
|   }) |   }) | ||||||
|   command!: string; |   command!: string; | ||||||
|  |  | ||||||
|  |   @IsOptional() | ||||||
|  |   @IsBoolean() | ||||||
|  |   includeAllAssets!: boolean; | ||||||
| } | } | ||||||
|   | |||||||
| @@ -21,12 +21,12 @@ export class JobController { | |||||||
|   @Put('/:jobId') |   @Put('/:jobId') | ||||||
|   async sendJobCommand( |   async sendJobCommand( | ||||||
|     @Param(ValidationPipe) params: GetJobDto, |     @Param(ValidationPipe) params: GetJobDto, | ||||||
|     @Body(ValidationPipe) body: JobCommandDto, |     @Body(ValidationPipe) dto: JobCommandDto, | ||||||
|   ): Promise<number> { |   ): Promise<number> { | ||||||
|     if (body.command === 'start') { |     if (dto.command === 'start') { | ||||||
|       return await this.jobService.start(params.jobId); |       return await this.jobService.start(params.jobId, dto.includeAllAssets); | ||||||
|     } |     } | ||||||
|     if (body.command === 'stop') { |     if (dto.command === 'stop') { | ||||||
|       return await this.jobService.stop(params.jobId); |       return await this.jobService.stop(params.jobId); | ||||||
|     } |     } | ||||||
|     return 0; |     return 0; | ||||||
|   | |||||||
| @@ -5,7 +5,7 @@ import { IAssetRepository } from '../asset/asset-repository'; | |||||||
| import { AssetType } from '@app/infra'; | import { AssetType } from '@app/infra'; | ||||||
| import { JobId } from './dto/get-job.dto'; | import { JobId } from './dto/get-job.dto'; | ||||||
| import { MACHINE_LEARNING_ENABLED } from '@app/common'; | import { MACHINE_LEARNING_ENABLED } from '@app/common'; | ||||||
|  | import { getFileNameWithoutExtension } from '../../utils/file-name.util'; | ||||||
| const jobIds = Object.values(JobId) as JobId[]; | const jobIds = Object.values(JobId) as JobId[]; | ||||||
|  |  | ||||||
| @Injectable() | @Injectable() | ||||||
| @@ -19,8 +19,8 @@ export class JobService { | |||||||
|     } |     } | ||||||
|   } |   } | ||||||
|  |  | ||||||
|   start(jobId: JobId): Promise<number> { |   start(jobId: JobId, includeAllAssets: boolean): Promise<number> { | ||||||
|     return this.run(this.asQueueName(jobId)); |     return this.run(this.asQueueName(jobId), includeAllAssets); | ||||||
|   } |   } | ||||||
|  |  | ||||||
|   async stop(jobId: JobId): Promise<number> { |   async stop(jobId: JobId): Promise<number> { | ||||||
| @@ -36,7 +36,7 @@ export class JobService { | |||||||
|     return response; |     return response; | ||||||
|   } |   } | ||||||
|  |  | ||||||
|   private async run(name: QueueName): Promise<number> { |   private async run(name: QueueName, includeAllAssets: boolean): Promise<number> { | ||||||
|     const isActive = await this.jobRepository.isActive(name); |     const isActive = await this.jobRepository.isActive(name); | ||||||
|     if (isActive) { |     if (isActive) { | ||||||
|       throw new BadRequestException(`Job is already running`); |       throw new BadRequestException(`Job is already running`); | ||||||
| @@ -44,7 +44,9 @@ export class JobService { | |||||||
|  |  | ||||||
|     switch (name) { |     switch (name) { | ||||||
|       case QueueName.VIDEO_CONVERSION: { |       case QueueName.VIDEO_CONVERSION: { | ||||||
|         const assets = await this._assetRepository.getAssetWithNoEncodedVideo(); |         const assets = includeAllAssets | ||||||
|  |           ? await this._assetRepository.getAllVideos() | ||||||
|  |           : await this._assetRepository.getAssetWithNoEncodedVideo(); | ||||||
|         for (const asset of assets) { |         for (const asset of assets) { | ||||||
|           await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } }); |           await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } }); | ||||||
|         } |         } | ||||||
| @@ -61,7 +63,10 @@ export class JobService { | |||||||
|           throw new BadRequestException('Machine learning is not enabled.'); |           throw new BadRequestException('Machine learning is not enabled.'); | ||||||
|         } |         } | ||||||
|  |  | ||||||
|         const assets = await this._assetRepository.getAssetWithNoSmartInfo(); |         const assets = includeAllAssets | ||||||
|  |           ? await this._assetRepository.getAll() | ||||||
|  |           : await this._assetRepository.getAssetWithNoSmartInfo(); | ||||||
|  |  | ||||||
|         for (const asset of assets) { |         for (const asset of assets) { | ||||||
|           await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } }); |           await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } }); | ||||||
|           await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } }); |           await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } }); | ||||||
| @@ -70,19 +75,37 @@ export class JobService { | |||||||
|       } |       } | ||||||
|  |  | ||||||
|       case QueueName.METADATA_EXTRACTION: { |       case QueueName.METADATA_EXTRACTION: { | ||||||
|         const assets = await this._assetRepository.getAssetWithNoEXIF(); |         const assets = includeAllAssets | ||||||
|  |           ? await this._assetRepository.getAll() | ||||||
|  |           : await this._assetRepository.getAssetWithNoEXIF(); | ||||||
|  |  | ||||||
|         for (const asset of assets) { |         for (const asset of assets) { | ||||||
|           if (asset.type === AssetType.VIDEO) { |           if (asset.type === AssetType.VIDEO) { | ||||||
|             await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } }); |             await this.jobRepository.add({ | ||||||
|  |               name: JobName.EXTRACT_VIDEO_METADATA, | ||||||
|  |               data: { | ||||||
|  |                 asset, | ||||||
|  |                 fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath), | ||||||
|  |               }, | ||||||
|  |             }); | ||||||
|           } else { |           } else { | ||||||
|             await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } }); |             await this.jobRepository.add({ | ||||||
|  |               name: JobName.EXIF_EXTRACTION, | ||||||
|  |               data: { | ||||||
|  |                 asset, | ||||||
|  |                 fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath), | ||||||
|  |               }, | ||||||
|  |             }); | ||||||
|           } |           } | ||||||
|         } |         } | ||||||
|         return assets.length; |         return assets.length; | ||||||
|       } |       } | ||||||
|  |  | ||||||
|       case QueueName.THUMBNAIL_GENERATION: { |       case QueueName.THUMBNAIL_GENERATION: { | ||||||
|         const assets = await this._assetRepository.getAssetWithNoThumbnail(); |         const assets = includeAllAssets | ||||||
|  |           ? await this._assetRepository.getAll() | ||||||
|  |           : await this._assetRepository.getAssetWithNoThumbnail(); | ||||||
|  |  | ||||||
|         for (const asset of assets) { |         for (const asset of assets) { | ||||||
|           await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } }); |           await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } }); | ||||||
|         } |         } | ||||||
|   | |||||||
| @@ -1,9 +1,8 @@ | |||||||
| import { Inject, Injectable, Logger } from '@nestjs/common'; | import { Inject, Injectable } from '@nestjs/common'; | ||||||
| import { Cron, CronExpression } from '@nestjs/schedule'; | import { Cron, CronExpression } from '@nestjs/schedule'; | ||||||
| import { InjectRepository } from '@nestjs/typeorm'; | import { InjectRepository } from '@nestjs/typeorm'; | ||||||
| import { IsNull, Not, Repository } from 'typeorm'; | import { IsNull, Not, Repository } from 'typeorm'; | ||||||
| import { AssetEntity, AssetType, ExifEntity, UserEntity } from '@app/infra'; | import { UserEntity } from '@app/infra'; | ||||||
| import { ConfigService } from '@nestjs/config'; |  | ||||||
| import { userUtils } from '@app/common'; | import { userUtils } from '@app/common'; | ||||||
| import { IJobRepository, JobName } from '@app/domain'; | import { IJobRepository, JobName } from '@app/domain'; | ||||||
|  |  | ||||||
| @@ -13,93 +12,8 @@ export class ScheduleTasksService { | |||||||
|     @InjectRepository(UserEntity) |     @InjectRepository(UserEntity) | ||||||
|     private userRepository: Repository<UserEntity>, |     private userRepository: Repository<UserEntity>, | ||||||
|  |  | ||||||
|     @InjectRepository(AssetEntity) |  | ||||||
|     private assetRepository: Repository<AssetEntity>, |  | ||||||
|  |  | ||||||
|     @InjectRepository(ExifEntity) |  | ||||||
|     private exifRepository: Repository<ExifEntity>, |  | ||||||
|  |  | ||||||
|     @Inject(IJobRepository) private jobRepository: IJobRepository, |     @Inject(IJobRepository) private jobRepository: IJobRepository, | ||||||
|  |  | ||||||
|     private configService: ConfigService, |  | ||||||
|   ) {} |   ) {} | ||||||
|  |  | ||||||
|   @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) |  | ||||||
|   async webpConversion() { |  | ||||||
|     const assets = await this.assetRepository.find({ |  | ||||||
|       where: { |  | ||||||
|         webpPath: '', |  | ||||||
|       }, |  | ||||||
|     }); |  | ||||||
|  |  | ||||||
|     if (assets.length == 0) { |  | ||||||
|       Logger.log('All assets has webp file - aborting task', 'CronjobWebpGenerator'); |  | ||||||
|       return; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     for (const asset of assets) { |  | ||||||
|       await this.jobRepository.add({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } }); |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
|  |  | ||||||
|   @Cron(CronExpression.EVERY_DAY_AT_1AM) |  | ||||||
|   async videoConversion() { |  | ||||||
|     const assets = await this.assetRepository.find({ |  | ||||||
|       where: { |  | ||||||
|         type: AssetType.VIDEO, |  | ||||||
|         mimeType: 'video/quicktime', |  | ||||||
|         encodedVideoPath: '', |  | ||||||
|       }, |  | ||||||
|       order: { |  | ||||||
|         createdAt: 'DESC', |  | ||||||
|       }, |  | ||||||
|     }); |  | ||||||
|  |  | ||||||
|     for (const asset of assets) { |  | ||||||
|       await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } }); |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
|  |  | ||||||
|   @Cron(CronExpression.EVERY_DAY_AT_2AM) |  | ||||||
|   async reverseGeocoding() { |  | ||||||
|     const isGeocodingEnabled = this.configService.get('DISABLE_REVERSE_GEOCODING') !== 'true'; |  | ||||||
|  |  | ||||||
|     if (isGeocodingEnabled) { |  | ||||||
|       const exifInfo = await this.exifRepository.find({ |  | ||||||
|         where: { |  | ||||||
|           city: IsNull(), |  | ||||||
|           longitude: Not(IsNull()), |  | ||||||
|           latitude: Not(IsNull()), |  | ||||||
|         }, |  | ||||||
|       }); |  | ||||||
|  |  | ||||||
|       for (const exif of exifInfo) { |  | ||||||
|         await this.jobRepository.add({ |  | ||||||
|           name: JobName.REVERSE_GEOCODING, |  | ||||||
|           // eslint-disable-next-line @typescript-eslint/no-non-null-assertion |  | ||||||
|           data: { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! }, |  | ||||||
|         }); |  | ||||||
|       } |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
|  |  | ||||||
|   @Cron(CronExpression.EVERY_DAY_AT_3AM) |  | ||||||
|   async extractExif() { |  | ||||||
|     const exifAssets = await this.assetRepository |  | ||||||
|       .createQueryBuilder('asset') |  | ||||||
|       .leftJoinAndSelect('asset.exifInfo', 'ei') |  | ||||||
|       .where('ei."assetId" IS NULL') |  | ||||||
|       .getMany(); |  | ||||||
|  |  | ||||||
|     for (const asset of exifAssets) { |  | ||||||
|       if (asset.type === AssetType.VIDEO) { |  | ||||||
|         await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } }); |  | ||||||
|       } else { |  | ||||||
|         await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } }); |  | ||||||
|       } |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
|  |  | ||||||
|   @Cron(CronExpression.EVERY_DAY_AT_11PM) |   @Cron(CronExpression.EVERY_DAY_AT_11PM) | ||||||
|   async deleteUserAndRelatedAssets() { |   async deleteUserAndRelatedAssets() { | ||||||
|     const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } }); |     const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } }); | ||||||
|   | |||||||
							
								
								
									
										5
									
								
								server/apps/immich/src/utils/file-name.util.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										5
									
								
								server/apps/immich/src/utils/file-name.util.ts
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,5 @@ | |||||||
|  | import { basename, extname } from 'node:path'; | ||||||
|  |  | ||||||
|  | export function getFileNameWithoutExtension(path: string): string { | ||||||
|  |   return basename(path, extname(path)); | ||||||
|  | } | ||||||
| @@ -216,7 +216,7 @@ export class MetadataExtractionProcessor { | |||||||
|         } |         } | ||||||
|       } |       } | ||||||
|  |  | ||||||
|       await this.exifRepository.save(newExif); |       await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); | ||||||
|     } catch (error: any) { |     } catch (error: any) { | ||||||
|       this.logger.error(`Error extracting EXIF ${error}`, error?.stack); |       this.logger.error(`Error extracting EXIF ${error}`, error?.stack); | ||||||
|     } |     } | ||||||
| @@ -327,7 +327,7 @@ export class MetadataExtractionProcessor { | |||||||
|         } |         } | ||||||
|       } |       } | ||||||
|  |  | ||||||
|       await this.exifRepository.save(newExif); |       await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); | ||||||
|       await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt }); |       await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt }); | ||||||
|     } catch (err) { |     } catch (err) { | ||||||
|       // do nothing |       // do nothing | ||||||
|   | |||||||
| @@ -11,6 +11,7 @@ import { Repository } from 'typeorm'; | |||||||
|  |  | ||||||
| @Processor(QueueName.VIDEO_CONVERSION) | @Processor(QueueName.VIDEO_CONVERSION) | ||||||
| export class VideoTranscodeProcessor { | export class VideoTranscodeProcessor { | ||||||
|  |   readonly logger = new Logger(VideoTranscodeProcessor.name); | ||||||
|   constructor( |   constructor( | ||||||
|     @InjectRepository(AssetEntity) |     @InjectRepository(AssetEntity) | ||||||
|     private assetRepository: Repository<AssetEntity>, |     private assetRepository: Repository<AssetEntity>, | ||||||
| @@ -20,7 +21,6 @@ export class VideoTranscodeProcessor { | |||||||
|   @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 }) |   @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 }) | ||||||
|   async videoConversion(job: Job<IVideoConversionProcessor>) { |   async videoConversion(job: Job<IVideoConversionProcessor>) { | ||||||
|     const { asset } = job.data; |     const { asset } = job.data; | ||||||
|  |  | ||||||
|     const basePath = APP_UPLOAD_LOCATION; |     const basePath = APP_UPLOAD_LOCATION; | ||||||
|     const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`; |     const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`; | ||||||
|  |  | ||||||
| @@ -30,17 +30,14 @@ export class VideoTranscodeProcessor { | |||||||
|  |  | ||||||
|     const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`; |     const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`; | ||||||
|  |  | ||||||
|     if (!asset.encodedVideoPath) { |  | ||||||
|       // Put the processing into its own async function to prevent the job exist right away |  | ||||||
|     await this.runVideoEncode(asset, savedEncodedPath); |     await this.runVideoEncode(asset, savedEncodedPath); | ||||||
|   } |   } | ||||||
|   } |  | ||||||
|  |  | ||||||
|   async runFFProbePipeline(asset: AssetEntity): Promise<FfprobeData> { |   async runFFProbePipeline(asset: AssetEntity): Promise<FfprobeData> { | ||||||
|     return new Promise((resolve, reject) => { |     return new Promise((resolve, reject) => { | ||||||
|       ffmpeg.ffprobe(asset.originalPath, (err, data) => { |       ffmpeg.ffprobe(asset.originalPath, (err, data) => { | ||||||
|         if (err || !data) { |         if (err || !data) { | ||||||
|           Logger.error(`Cannot probe video ${err}`, 'mp4Conversion'); |           this.logger.error(`Cannot probe video ${err}`, 'runFFProbePipeline'); | ||||||
|           reject(err); |           reject(err); | ||||||
|         } |         } | ||||||
|  |  | ||||||
| @@ -88,14 +85,14 @@ export class VideoTranscodeProcessor { | |||||||
|         ]) |         ]) | ||||||
|         .output(savedEncodedPath) |         .output(savedEncodedPath) | ||||||
|         .on('start', () => { |         .on('start', () => { | ||||||
|           Logger.log('Start Converting Video', 'mp4Conversion'); |           this.logger.log('Start Converting Video'); | ||||||
|         }) |         }) | ||||||
|         .on('error', (error) => { |         .on('error', (error) => { | ||||||
|           Logger.error(`Cannot Convert Video ${error}`, 'mp4Conversion'); |           this.logger.error(`Cannot Convert Video ${error}`); | ||||||
|           reject(); |           reject(); | ||||||
|         }) |         }) | ||||||
|         .on('end', async () => { |         .on('end', async () => { | ||||||
|           Logger.log(`Converting Success ${asset.id}`, 'mp4Conversion'); |           this.logger.log(`Converting Success ${asset.id}`); | ||||||
|           await this.assetRepository.update({ id: asset.id }, { encodedVideoPath: savedEncodedPath }); |           await this.assetRepository.update({ id: asset.id }, { encodedVideoPath: savedEncodedPath }); | ||||||
|           resolve(); |           resolve(); | ||||||
|         }) |         }) | ||||||
|   | |||||||
| @@ -4538,10 +4538,14 @@ | |||||||
|         "properties": { |         "properties": { | ||||||
|           "command": { |           "command": { | ||||||
|             "$ref": "#/components/schemas/JobCommand" |             "$ref": "#/components/schemas/JobCommand" | ||||||
|  |           }, | ||||||
|  |           "includeAllAssets": { | ||||||
|  |             "type": "boolean" | ||||||
|           } |           } | ||||||
|         }, |         }, | ||||||
|         "required": [ |         "required": [ | ||||||
|           "command" |           "command", | ||||||
|  |           "includeAllAssets" | ||||||
|         ] |         ] | ||||||
|       } |       } | ||||||
|     } |     } | ||||||
|   | |||||||
							
								
								
									
										6
									
								
								web/src/api/open-api/api.ts
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										6
									
								
								web/src/api/open-api/api.ts
									
									
									
										generated
									
									
									
								
							| @@ -1203,6 +1203,12 @@ export interface JobCommandDto { | |||||||
|      * @memberof JobCommandDto |      * @memberof JobCommandDto | ||||||
|      */ |      */ | ||||||
|     'command': JobCommand; |     'command': JobCommand; | ||||||
|  |     /** | ||||||
|  |      *  | ||||||
|  |      * @type {boolean} | ||||||
|  |      * @memberof JobCommandDto | ||||||
|  |      */ | ||||||
|  |     'includeAllAssets': boolean; | ||||||
| } | } | ||||||
| /** | /** | ||||||
|  *  |  *  | ||||||
|   | |||||||
| @@ -101,4 +101,8 @@ input:focus-visible { | |||||||
| 		display: none; | 		display: none; | ||||||
| 		scrollbar-width: none; | 		scrollbar-width: none; | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
|  | 	.job-play-button { | ||||||
|  | 		@apply h-full flex flex-col place-items-center place-content-center px-8 text-gray-600 transition-all hover:bg-immich-primary hover:text-white dark:text-gray-200 dark:hover:bg-immich-dark-primary text-sm dark:hover:text-black w-[120px] gap-2; | ||||||
|  | 	} | ||||||
| } | } | ||||||
|   | |||||||
| @@ -1,76 +1,102 @@ | |||||||
| <script lang="ts"> | <script lang="ts"> | ||||||
| 	import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte'; | 	import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte'; | ||||||
|  | 	import SelectionSearch from 'svelte-material-icons/SelectionSearch.svelte'; | ||||||
|  | 	import Play from 'svelte-material-icons/Play.svelte'; | ||||||
|  | 	import AllInclusive from 'svelte-material-icons/AllInclusive.svelte'; | ||||||
|  |  | ||||||
| 	import { createEventDispatcher } from 'svelte'; | 	import { createEventDispatcher } from 'svelte'; | ||||||
| 	import { JobCounts } from '@api'; | 	import { JobCounts } from '@api'; | ||||||
|  |  | ||||||
| 	export let title: string; | 	export let title: string; | ||||||
| 	export let subtitle: string; | 	export let subtitle: string; | ||||||
| 	export let buttonTitle = 'Run'; |  | ||||||
| 	export let jobCounts: JobCounts; | 	export let jobCounts: JobCounts; | ||||||
|  | 	/** | ||||||
|  | 	 * Show options to run job on all assets of just missing ones | ||||||
|  | 	 */ | ||||||
|  | 	export let showOptions = true; | ||||||
|  |  | ||||||
|  | 	$: isRunning = jobCounts.active > 0 || jobCounts.waiting > 0; | ||||||
|  |  | ||||||
| 	const dispatch = createEventDispatcher(); | 	const dispatch = createEventDispatcher(); | ||||||
|  |  | ||||||
|  | 	const run = (includeAllAssets: boolean) => { | ||||||
|  | 		dispatch('click', { includeAllAssets }); | ||||||
|  | 	}; | ||||||
| </script> | </script> | ||||||
|  |  | ||||||
| <div class="flex border-b pb-5 dark:border-b-immich-dark-gray"> | <div class="flex justify-between rounded-3xl bg-gray-100 dark:bg-immich-dark-gray"> | ||||||
| 	<div class="w-[70%]"> | 	<div id="job-info" class="w-[70%] p-9"> | ||||||
| 		<h1 class="text-immich-primary dark:text-immich-dark-primary text-sm font-semibold"> | 		<div class="flex flex-col gap-2"> | ||||||
|  | 			<div class="text-xl font-semibold text-immich-primary dark:text-immich-dark-primary"> | ||||||
| 				{title.toUpperCase()} | 				{title.toUpperCase()} | ||||||
| 		</h1> | 			</div> | ||||||
| 		<p class="text-sm mt-1 dark:text-immich-dark-fg">{subtitle}</p> |  | ||||||
| 		<p class="text-sm dark:text-immich-dark-fg"> | 			{#if subtitle.length > 0} | ||||||
| 			<slot /> | 				<div class="text-sm dark:text-white">{subtitle}</div> | ||||||
| 		</p> |  | ||||||
| 		<table class="text-left w-full mt-5"> |  | ||||||
| 			<!-- table header --> |  | ||||||
| 			<thead |  | ||||||
| 				class="border rounded-md mb-2 dark:bg-immich-dark-gray dark:border-immich-dark-gray bg-immich-primary/10 flex text-immich-primary dark:text-immich-dark-primary w-full h-12" |  | ||||||
| 			> |  | ||||||
| 				<tr class="flex w-full place-items-center"> |  | ||||||
| 					<th class="text-center w-1/3 font-medium text-sm">Status</th> |  | ||||||
| 					<th class="text-center w-1/3 font-medium text-sm">Active</th> |  | ||||||
| 					<th class="text-center w-1/3 font-medium text-sm">Waiting</th> |  | ||||||
| 				</tr> |  | ||||||
| 			</thead> |  | ||||||
| 			<tbody |  | ||||||
| 				class="overflow-y-auto rounded-md w-full max-h-[320px] block border bg-white dark:border-immich-dark-gray dark:bg-immich-dark-gray/75 dark:text-immich-dark-fg" |  | ||||||
| 			> |  | ||||||
| 				<tr class="text-center flex place-items-center w-full h-[60px]"> |  | ||||||
| 					<td class="text-sm px-2 w-1/3 text-ellipsis"> |  | ||||||
| 						{#if jobCounts} |  | ||||||
| 							<span>{jobCounts.active > 0 || jobCounts.waiting > 0 ? 'Active' : 'Idle'}</span> |  | ||||||
| 						{:else} |  | ||||||
| 							<LoadingSpinner /> |  | ||||||
| 			{/if} | 			{/if} | ||||||
| 					</td> | 			<div class="text-sm dark:text-white"><slot /></div> | ||||||
| 					<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis"> |  | ||||||
|  | 			<div class="flex w-full mt-4"> | ||||||
|  | 				<div | ||||||
|  | 					class="flex place-items-center justify-between bg-immich-primary dark:bg-immich-dark-primary text-white dark:text-immich-dark-gray w-full rounded-tl-lg rounded-bl-lg py-4 pl-4 pr-6" | ||||||
|  | 				> | ||||||
|  | 					<p>Active</p> | ||||||
|  | 					<p class="text-2xl"> | ||||||
| 						{#if jobCounts.active !== undefined} | 						{#if jobCounts.active !== undefined} | ||||||
| 							{jobCounts.active} | 							{jobCounts.active} | ||||||
| 						{:else} | 						{:else} | ||||||
| 							<LoadingSpinner /> | 							<LoadingSpinner /> | ||||||
| 						{/if} | 						{/if} | ||||||
| 					</td> | 					</p> | ||||||
| 					<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis"> | 				</div> | ||||||
|  |  | ||||||
|  | 				<div | ||||||
|  | 					class="flex place-items-center justify-between bg-gray-200 text-immich-dark-bg dark:bg-gray-700 dark:text-immich-gray w-full rounded-tr-lg rounded-br-lg py-4 pr-4 pl-6" | ||||||
|  | 				> | ||||||
|  | 					<p class="text-2xl"> | ||||||
| 						{#if jobCounts.waiting !== undefined} | 						{#if jobCounts.waiting !== undefined} | ||||||
| 							{jobCounts.waiting} | 							{jobCounts.waiting} | ||||||
| 						{:else} | 						{:else} | ||||||
| 							<LoadingSpinner /> | 							<LoadingSpinner /> | ||||||
| 						{/if} | 						{/if} | ||||||
| 					</td> | 					</p> | ||||||
| 				</tr> | 					<p>Waiting</p> | ||||||
| 			</tbody> |  | ||||||
| 		</table> |  | ||||||
| 				</div> | 				</div> | ||||||
| 	<div class="w-[30%] flex place-items-center place-content-end"> | 			</div> | ||||||
|  | 		</div> | ||||||
|  | 	</div> | ||||||
|  | 	<div id="job-action" class="flex flex-col"> | ||||||
|  | 		{#if isRunning} | ||||||
| 			<button | 			<button | ||||||
| 			on:click={() => dispatch('click')} | 				class="job-play-button bg-gray-300/90 dark:bg-gray-600/90 rounded-br-3xl rounded-tr-3xl disabled:cursor-not-allowed" | ||||||
| 			class="px-6 py-3 text-sm bg-immich-primary dark:bg-immich-dark-primary font-medium rounded-2xl hover:bg-immich-primary/50 transition-all hover:cursor-pointer disabled:cursor-not-allowed shadow-sm text-immich-bg dark:text-immich-dark-gray" | 				disabled | ||||||
| 			disabled={jobCounts.active > 0 && jobCounts.waiting > 0} |  | ||||||
| 			> | 			> | ||||||
| 			{#if jobCounts.active > 0 || jobCounts.waiting > 0} |  | ||||||
| 				<LoadingSpinner /> | 				<LoadingSpinner /> | ||||||
| 			{:else} |  | ||||||
| 				{buttonTitle} |  | ||||||
| 			{/if} |  | ||||||
| 			</button> | 			</button> | ||||||
|  | 		{/if} | ||||||
|  |  | ||||||
|  | 		{#if !isRunning} | ||||||
|  | 			{#if showOptions} | ||||||
|  | 				<button | ||||||
|  | 					class="job-play-button bg-gray-300 dark:bg-gray-600 rounded-tr-3xl" | ||||||
|  | 					on:click={() => run(true)} | ||||||
|  | 				> | ||||||
|  | 					<AllInclusive size="18" /> ALL | ||||||
|  | 				</button> | ||||||
|  | 				<button | ||||||
|  | 					class="job-play-button bg-gray-300/90 dark:bg-gray-600/90 rounded-br-3xl" | ||||||
|  | 					on:click={() => run(false)} | ||||||
|  | 				> | ||||||
|  | 					<SelectionSearch size="18" /> MISSING | ||||||
|  | 				</button> | ||||||
|  | 			{:else} | ||||||
|  | 				<button | ||||||
|  | 					class="job-play-button bg-gray-300/90 dark:bg-gray-600/90 rounded-br-3xl rounded-tr-3xl" | ||||||
|  | 					on:click={() => run(true)} | ||||||
|  | 				> | ||||||
|  | 					<Play size="48" /> | ||||||
|  | 				</button> | ||||||
|  | 			{/if} | ||||||
|  | 		{/if} | ||||||
| 	</div> | 	</div> | ||||||
| </div> | </div> | ||||||
|   | |||||||
| @@ -18,20 +18,28 @@ | |||||||
|  |  | ||||||
| 	onMount(async () => { | 	onMount(async () => { | ||||||
| 		await load(); | 		await load(); | ||||||
| 		timer = setInterval(async () => await load(), 5_000); | 		timer = setInterval(async () => await load(), 1_000); | ||||||
| 	}); | 	}); | ||||||
|  |  | ||||||
| 	onDestroy(() => { | 	onDestroy(() => { | ||||||
| 		clearInterval(timer); | 		clearInterval(timer); | ||||||
| 	}); | 	}); | ||||||
|  |  | ||||||
| 	const run = async (jobId: JobId, jobName: string, emptyMessage: string) => { | 	const run = async ( | ||||||
|  | 		jobId: JobId, | ||||||
|  | 		jobName: string, | ||||||
|  | 		emptyMessage: string, | ||||||
|  | 		includeAllAssets: boolean | ||||||
|  | 	) => { | ||||||
| 		try { | 		try { | ||||||
| 			const { data } = await api.jobApi.sendJobCommand(jobId, { command: JobCommand.Start }); | 			const { data } = await api.jobApi.sendJobCommand(jobId, { | ||||||
|  | 				command: JobCommand.Start, | ||||||
|  | 				includeAllAssets | ||||||
|  | 			}); | ||||||
|  |  | ||||||
| 			if (data) { | 			if (data) { | ||||||
| 				notificationController.show({ | 				notificationController.show({ | ||||||
| 					message: `Started ${jobName}`, | 					message: includeAllAssets ? `Started ${jobName} for all assets` : `Started ${jobName}`, | ||||||
| 					type: NotificationType.Info | 					type: NotificationType.Info | ||||||
| 				}); | 				}); | ||||||
| 			} else { | 			} else { | ||||||
| @@ -43,53 +51,77 @@ | |||||||
| 	}; | 	}; | ||||||
| </script> | </script> | ||||||
|  |  | ||||||
| <div class="flex flex-col gap-10"> | <div class="flex flex-col gap-7"> | ||||||
| 	{#if jobs} | 	{#if jobs} | ||||||
| 		<JobTile | 		<JobTile | ||||||
| 			title={'Generate thumbnails'} | 			title={'Generate thumbnails'} | ||||||
| 			subtitle={'Regenerate missing thumbnail (JPEG, WEBP)'} | 			subtitle={'Regenerate JPEG and WebP thumbnails'} | ||||||
| 			on:click={() => | 			on:click={(e) => { | ||||||
| 				run(JobId.ThumbnailGeneration, 'thumbnail generation', 'No missing thumbnails found')} | 				const { includeAllAssets } = e.detail; | ||||||
|  |  | ||||||
|  | 				run( | ||||||
|  | 					JobId.ThumbnailGeneration, | ||||||
|  | 					'thumbnail generation', | ||||||
|  | 					'No missing thumbnails found', | ||||||
|  | 					includeAllAssets | ||||||
|  | 				); | ||||||
|  | 			}} | ||||||
| 			jobCounts={jobs[JobId.ThumbnailGeneration]} | 			jobCounts={jobs[JobId.ThumbnailGeneration]} | ||||||
| 		/> | 		/> | ||||||
|  |  | ||||||
| 		<JobTile | 		<JobTile | ||||||
| 			title={'Extract EXIF'} | 			title={'EXTRACT METADATA'} | ||||||
| 			subtitle={'Extract missing EXIF information'} | 			subtitle={'Extract metadata information i.e. GPS, resolution...etc'} | ||||||
| 			on:click={() => run(JobId.MetadataExtraction, 'extract EXIF', 'No missing EXIF found')} | 			on:click={(e) => { | ||||||
|  | 				const { includeAllAssets } = e.detail; | ||||||
|  | 				run(JobId.MetadataExtraction, 'extract EXIF', 'No missing EXIF found', includeAllAssets); | ||||||
|  | 			}} | ||||||
| 			jobCounts={jobs[JobId.MetadataExtraction]} | 			jobCounts={jobs[JobId.MetadataExtraction]} | ||||||
| 		/> | 		/> | ||||||
|  |  | ||||||
| 		<JobTile | 		<JobTile | ||||||
| 			title={'Detect objects'} | 			title={'Detect objects'} | ||||||
| 			subtitle={'Run machine learning process to detect and classify objects'} | 			subtitle={'Run machine learning process to detect and classify objects'} | ||||||
| 			on:click={() => | 			on:click={(e) => { | ||||||
| 				run(JobId.MachineLearning, 'object detection', 'No missing object detection found')} | 				const { includeAllAssets } = e.detail; | ||||||
|  |  | ||||||
|  | 				run( | ||||||
|  | 					JobId.MachineLearning, | ||||||
|  | 					'object detection', | ||||||
|  | 					'No missing object detection found', | ||||||
|  | 					includeAllAssets | ||||||
|  | 				); | ||||||
|  | 			}} | ||||||
| 			jobCounts={jobs[JobId.MachineLearning]} | 			jobCounts={jobs[JobId.MachineLearning]} | ||||||
| 		> | 		> | ||||||
| 			Note that some assets may not have any objects detected, this is normal. | 			Note that some assets may not have any objects detected | ||||||
| 		</JobTile> | 		</JobTile> | ||||||
|  |  | ||||||
| 		<JobTile | 		<JobTile | ||||||
| 			title={'Video transcoding'} | 			title={'Video transcoding'} | ||||||
| 			subtitle={'Run video transcoding process to transcode videos not in the desired format'} | 			subtitle={'Transcode videos not in the desired format'} | ||||||
| 			on:click={() => | 			on:click={(e) => { | ||||||
|  | 				const { includeAllAssets } = e.detail; | ||||||
| 				run( | 				run( | ||||||
| 					JobId.VideoConversion, | 					JobId.VideoConversion, | ||||||
| 					'video conversion', | 					'video conversion', | ||||||
| 					'No videos without an encoded version found' | 					'No videos without an encoded version found', | ||||||
| 				)} | 					includeAllAssets | ||||||
|  | 				); | ||||||
|  | 			}} | ||||||
| 			jobCounts={jobs[JobId.VideoConversion]} | 			jobCounts={jobs[JobId.VideoConversion]} | ||||||
| 		/> | 		/> | ||||||
|  |  | ||||||
| 		<JobTile | 		<JobTile | ||||||
| 			title={'Storage migration'} | 			title={'Storage migration'} | ||||||
|  | 			showOptions={false} | ||||||
| 			subtitle={''} | 			subtitle={''} | ||||||
| 			on:click={() => | 			on:click={() => | ||||||
| 				run( | 				run( | ||||||
| 					JobId.StorageTemplateMigration, | 					JobId.StorageTemplateMigration, | ||||||
| 					'storage template migration', | 					'storage template migration', | ||||||
| 					'All files have been migrated to the new storage template' | 					'All files have been migrated to the new storage template', | ||||||
|  | 					false | ||||||
| 				)} | 				)} | ||||||
| 			jobCounts={jobs[JobId.StorageTemplateMigration]} | 			jobCounts={jobs[JobId.StorageTemplateMigration]} | ||||||
| 		> | 		> | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user