KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-12-08 04:09:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): jobs (#2023)

Browse Source 
* refactor: job to domain

* chore: regenerate open api

* chore: tests

* fix: missing breaks

* fix: get asset with missing exif data

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jason Rasmussen
2023-03-20 11:55:28 -04:00
committed by GitHub
parent db6b14361d
commit 386eef046d
68 changed files with 1355 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
server/apps/immich/src/api-v1/asset/asset-repository.ts
View File

@@ -38,10 +38,6 @@ export interface IAssetRepository {
getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
getAssetWithNoThumbnail(): Promise<AssetEntity[]>;
getAssetWithNoEncodedVideo(): Promise<AssetEntity[]>;
getAssetWithNoEXIF(): Promise<AssetEntity[]>;
getAssetWithNoSmartInfo(): Promise<AssetEntity[]>;
getExistingAssets(
userId: string,
checkDuplicateAssetDto: CheckExistingAssetsDto,
@@ -76,45 +72,6 @@ export class AssetRepository implements IAssetRepository {
});
}
async getAssetWithNoSmartInfo(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.smartInfo', 'si')
.where('asset.resizePath IS NOT NULL')
.andWhere('si.assetId IS NULL')
.andWhere('asset.isVisible = true')
.getMany();
}
async getAssetWithNoThumbnail(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: [
{ resizePath: IsNull(), isVisible: true },
{ resizePath: '', isVisible: true },
{ webpPath: IsNull(), isVisible: true },
{ webpPath: '', isVisible: true },
],
});
}
async getAssetWithNoEncodedVideo(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: [
{ type: AssetType.VIDEO, encodedVideoPath: IsNull() },
{ type: AssetType.VIDEO, encodedVideoPath: '' },
],
});
}
async getAssetWithNoEXIF(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'ei')
.where('ei."assetId" IS NULL')
.andWhere('asset.isVisible = true')
.getMany();
}
async getAssetCountByUserId(ownerId: string): Promise<AssetCountByUserIdResponseDto> {
// Get asset count by AssetType
const items = await this.assetRepository

4
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@@ -146,10 +146,6 @@ describe('AssetService', () => {
getAssetByTimeBucket: jest.fn(),
getAssetByChecksum: jest.fn(),
getAssetCountByUserId: jest.fn(),
getAssetWithNoEXIF: jest.fn(),
getAssetWithNoThumbnail: jest.fn(),
getAssetWithNoSmartInfo: jest.fn(),
getAssetWithNoEncodedVideo: jest.fn(),
getExistingAssets: jest.fn(),
countByIdAndUser: jest.fn(),
};

2
server/apps/immich/src/api-v1/asset/asset.service.ts
View File

@@ -63,7 +63,7 @@ import { AssetSearchDto } from './dto/asset-search.dto';
import { AddAssetsDto } from '../album/dto/add-assets.dto';
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
import path from 'path';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
import { getFileNameWithoutExtension } from '@app/domain';
const fileInfo = promisify(stat);

23
server/apps/immich/src/api-v1/job/dto/get-job.dto.ts
View File

@@ -1,23 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsEnum, IsNotEmpty } from 'class-validator';
export enum JobId {
THUMBNAIL_GENERATION = 'thumbnail-generation',
METADATA_EXTRACTION = 'metadata-extraction',
VIDEO_CONVERSION = 'video-conversion',
MACHINE_LEARNING = 'machine-learning',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
}
export class GetJobDto {
@IsNotEmpty()
@IsEnum(JobId, {
message: `params must be one of ${Object.values(JobId).join()}`,
})
@ApiProperty({
type: String,
enum: JobId,
enumName: 'JobId',
})
jobId!: JobId;
}

16
server/apps/immich/src/api-v1/job/dto/job-command.dto.ts
View File

@@ -1,16 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsBoolean, IsIn, IsNotEmpty, IsOptional } from 'class-validator';
export class JobCommandDto {
@IsNotEmpty()
@IsIn(['start', 'stop'])
@ApiProperty({
enum: ['start', 'stop'],
enumName: 'JobCommand',
})
command!: string;
@IsOptional()
@IsBoolean()
includeAllAssets!: boolean;
}

33
server/apps/immich/src/api-v1/job/job.controller.ts
View File

@@ -1,33 +0,0 @@
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Authenticated } from '../../decorators/authenticated.decorator';
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
import { GetJobDto } from './dto/get-job.dto';
import { JobService } from './job.service';
import { JobCommandDto } from './dto/job-command.dto';
@Authenticated({ admin: true })
@ApiTags('Job')
@Controller('jobs')
export class JobController {
constructor(private readonly jobService: JobService) {}
@Get()
getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
return this.jobService.getAllJobsStatus();
}
@Put('/:jobId')
async sendJobCommand(
@Param(ValidationPipe) params: GetJobDto,
@Body(ValidationPipe) dto: JobCommandDto,
): Promise<number> {
if (dto.command === 'start') {
return await this.jobService.start(params.jobId, dto.includeAllAssets);
}
if (dto.command === 'stop') {
return await this.jobService.stop(params.jobId);
}
return 0;
}
}

11
server/apps/immich/src/api-v1/job/job.module.ts
View File

@@ -1,11 +0,0 @@
import { Module } from '@nestjs/common';
import { JobService } from './job.service';
import { JobController } from './job.controller';
import { AssetModule } from '../asset/asset.module';
@Module({
imports: [AssetModule],
controllers: [JobController],
providers: [JobService],
})
export class JobModule {}

142
server/apps/immich/src/api-v1/job/job.service.ts
View File

@@ -1,142 +0,0 @@
import { JobName, IJobRepository, QueueName } from '@app/domain';
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
import { IAssetRepository } from '../asset/asset-repository';
import { AssetType } from '@app/infra';
import { JobId } from './dto/get-job.dto';
import { MACHINE_LEARNING_ENABLED } from '@app/common';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
const jobIds = Object.values(JobId) as JobId[];
@Injectable()
export class JobService {
constructor(
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
for (const jobId of jobIds) {
this.jobRepository.empty(this.asQueueName(jobId));
}
}
start(jobId: JobId, includeAllAssets: boolean): Promise<number> {
return this.run(this.asQueueName(jobId), includeAllAssets);
}
async stop(jobId: JobId): Promise<number> {
await this.jobRepository.empty(this.asQueueName(jobId));
return 0;
}
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
const response = new AllJobStatusResponseDto();
for (const jobId of jobIds) {
response[jobId] = await this.jobRepository.getJobCounts(this.asQueueName(jobId));
}
return response;
}
private async run(name: QueueName, includeAllAssets: boolean): Promise<number> {
const isActive = await this.jobRepository.isActive(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
}
switch (name) {
case QueueName.VIDEO_CONVERSION: {
const assets = includeAllAssets
? await this._assetRepository.getAllVideos()
: await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
return assets.length;
}
case QueueName.STORAGE_TEMPLATE_MIGRATION:
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
return 1;
case QueueName.MACHINE_LEARNING: {
if (!MACHINE_LEARNING_ENABLED) {
throw new BadRequestException('Machine learning is not enabled.');
}
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
}
return assets.length;
}
case QueueName.METADATA_EXTRACTION: {
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoEXIF();
for (const asset of assets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.queue({
name: JobName.EXTRACT_VIDEO_METADATA,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
} else {
await this.jobRepository.queue({
name: JobName.EXIF_EXTRACTION,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
}
}
return assets.length;
}
case QueueName.THUMBNAIL_GENERATION: {
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}
return assets.length;
}
default:
return 0;
}
}
private asQueueName(jobId: JobId) {
switch (jobId) {
case JobId.THUMBNAIL_GENERATION:
return QueueName.THUMBNAIL_GENERATION;
case JobId.METADATA_EXTRACTION:
return QueueName.METADATA_EXTRACTION;
case JobId.VIDEO_CONVERSION:
return QueueName.VIDEO_CONVERSION;
case JobId.STORAGE_TEMPLATE_MIGRATION:
return QueueName.STORAGE_TEMPLATE_MIGRATION;
case JobId.MACHINE_LEARNING:
return QueueName.MACHINE_LEARNING;
default:
throw new BadRequestException(`Invalid job id: ${jobId}`);
}
}
}

32
server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts
View File

@@ -1,32 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { JobId } from '../dto/get-job.dto';
export class JobCounts {
@ApiProperty({ type: 'integer' })
active!: number;
@ApiProperty({ type: 'integer' })
completed!: number;
@ApiProperty({ type: 'integer' })
failed!: number;
@ApiProperty({ type: 'integer' })
delayed!: number;
@ApiProperty({ type: 'integer' })
waiting!: number;
}
export class AllJobStatusResponseDto {
@ApiProperty({ type: JobCounts })
[JobId.THUMBNAIL_GENERATION]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.METADATA_EXTRACTION]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.VIDEO_CONVERSION]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.MACHINE_LEARNING]!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.STORAGE_TEMPLATE_MIGRATION]!: JobCounts;
}

5
server/apps/immich/src/app.module.ts
View File

@@ -7,7 +7,6 @@ import { AlbumModule } from './api-v1/album/album.module';
import { AppController } from './app.controller';
import { ScheduleModule } from '@nestjs/schedule';
import { ScheduleTasksModule } from './modules/schedule-tasks/schedule-tasks.module';
import { JobModule } from './api-v1/job/job.module';
import { TagModule } from './api-v1/tag/tag.module';
import { DomainModule, SearchService } from '@app/domain';
import { InfraModule } from '@app/infra';
@@ -15,6 +14,7 @@ import {
APIKeyController,
AuthController,
DeviceInfoController,
JobController,
OAuthController,
SearchController,
ShareController,
@@ -42,8 +42,6 @@ import { AuthGuard } from './middlewares/auth.guard';
ScheduleTasksModule,
JobModule,
TagModule,
],
controllers: [
@@ -51,6 +49,7 @@ import { AuthGuard } from './middlewares/auth.guard';
APIKeyController,
AuthController,
DeviceInfoController,
JobController,
OAuthController,
SearchController,
ShareController,

1
server/apps/immich/src/controllers/index.ts
View File

@@ -1,6 +1,7 @@
export * from './api-key.controller';
export * from './auth.controller';
export * from './device-info.controller';
export * from './job.controller';
export * from './oauth.controller';
export * from './search.controller';
export * from './share.controller';

21
server/apps/immich/src/controllers/job.controller.ts Normal file
View File

@@ -0,0 +1,21 @@
import { AllJobStatusResponseDto, JobCommandDto, JobIdDto, JobService } from '@app/domain';
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Authenticated } from '../decorators/authenticated.decorator';
@Authenticated({ admin: true })
@ApiTags('Job')
@Controller('jobs')
export class JobController {
constructor(private readonly jobService: JobService) {}
@Get()
getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
return this.jobService.getAllJobsStatus();
}
@Put('/:jobId')
sendJobCommand(@Param(ValidationPipe) { jobId }: JobIdDto, @Body(ValidationPipe) dto: JobCommandDto): Promise<void> {
return this.jobService.handleCommand(jobId, dto);
}
}

5
server/apps/immich/src/utils/file-name.util.ts
View File

@@ -1,5 +0,0 @@
import { basename, extname } from 'node:path';
export function getFileNameWithoutExtension(path: string): string {
return basename(path, extname(path));
}