KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-12-31 14:26:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server,web): libraries (#3124)

Browse Source 
* feat: libraries

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors
2023-09-20 13:16:33 +02:00
committed by GitHub
parent 816db700e1
commit acdc66413c
143 changed files with 10941 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
server/src/domain/job/job.constants.ts
View File

@@ -9,6 +9,7 @@ export enum QueueName {
STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
SEARCH = 'search',
SIDECAR = 'sidecar',
LIBRARY = 'library',
}
export enum JobCommand {
@@ -53,6 +54,15 @@ export enum JobName {
RECOGNIZE_FACES = 'recognize-faces',
PERSON_CLEANUP = 'person-cleanup',
// library managment
LIBRARY_SCAN = 'library-refresh',
LIBRARY_SCAN_ASSET = 'library-refresh-asset',
LIBRARY_REMOVE_OFFLINE = 'library-remove-offline',
LIBRARY_MARK_ASSET_OFFLINE = 'library-mark-asset-offline',
LIBRARY_DELETE = 'library-delete',
LIBRARY_QUEUE_SCAN_ALL = 'library-queue-all-refresh',
LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup',
// cleanup
DELETE_FILES = 'delete-files',
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
@@ -140,4 +150,13 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,
[JobName.SIDECAR_SYNC]: QueueName.SIDECAR,
// Library managment
[JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY,
[JobName.LIBRARY_MARK_ASSET_OFFLINE]: QueueName.LIBRARY,
[JobName.LIBRARY_SCAN]: QueueName.LIBRARY,
[JobName.LIBRARY_DELETE]: QueueName.LIBRARY,
[JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY,
};

3
server/src/domain/job/job.dto.ts
View File

@@ -79,4 +79,7 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
@ApiProperty({ type: JobStatusDto })
[QueueName.SIDECAR]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.LIBRARY]!: JobStatusDto;
}

15
server/src/domain/job/job.interface.ts
View File

@@ -22,6 +22,21 @@ export interface IEntityJob extends IBaseJob {
source?: 'upload';
}
export interface IOfflineLibraryFileJob extends IEntityJob {
assetPath: string;
}
export interface ILibraryFileJob extends IEntityJob {
ownerId: string;
assetPath: string;
forceRefresh: boolean;
}
export interface ILibraryRefreshJob extends IEntityJob {
refreshModifiedFiles: boolean;
refreshAllFiles: boolean;
}
export interface IBulkEntityJob extends IBaseJob {
ids: string[];
}

13
server/src/domain/job/job.repository.ts
View File

@@ -1,4 +1,5 @@
import { JobName, QueueName } from './job.constants';
import {
IAssetFaceJob,
IBaseJob,
@@ -6,6 +7,9 @@ import {
IDeleteFilesJob,
IEntityJob,
IFaceThumbnailJob,
ILibraryFileJob,
ILibraryRefreshJob,
IOfflineLibraryFileJob,
} from './job.interface';
export interface JobCounts {
@@ -74,6 +78,15 @@ export type JobItem =
// Asset Deletion
| { name: JobName.PERSON_CLEANUP; data?: IBaseJob }
// Library Managment
| { name: JobName.LIBRARY_SCAN_ASSET; data: ILibraryFileJob }
| { name: JobName.LIBRARY_MARK_ASSET_OFFLINE; data: IOfflineLibraryFileJob }
| { name: JobName.LIBRARY_SCAN; data: ILibraryRefreshJob }
| { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
| { name: JobName.LIBRARY_DELETE; data: IEntityJob }
| { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }
// Search
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }

4
server/src/domain/job/job.service.spec.ts
View File

@@ -52,6 +52,7 @@ describe(JobService.name, () => {
[{ name: JobName.PERSON_CLEANUP }],
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
[{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }],
]);
});
});
@@ -97,6 +98,7 @@ describe(JobService.name, () => {
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
[QueueName.SIDECAR]: expectedJobStatus,
[QueueName.LIBRARY]: expectedJobStatus,
});
});
});
@@ -225,6 +227,7 @@ describe(JobService.name, () => {
[QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
[QueueName.SEARCH]: { concurrency: 10 },
[QueueName.SIDECAR]: { concurrency: 10 },
[QueueName.LIBRARY]: { concurrency: 10 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
@@ -237,6 +240,7 @@ describe(JobService.name, () => {
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.OBJECT_TAGGING, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);

6
server/src/domain/job/job.service.ts
View File

@@ -98,6 +98,9 @@ export class JobService {
await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } });
case QueueName.LIBRARY:
return this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force } });
default:
throw new BadRequestException(`Invalid job name: ${name}`);
}
@@ -118,7 +121,7 @@ export class JobService {
await this.onDone(item);
}
} catch (error: Error | any) {
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, data);
this.logger.error(`Unable to run job handler (${queueName}/${name}): ${error}`, error?.stack, data);
}
});
}
@@ -138,6 +141,7 @@ export class JobService {
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } });
}
/**