KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(web/server): Add options to rerun job on all assets (#1422)

Browse Source
This commit is contained in:
Alex
2023-01-26 22:50:22 -06:00
committed by GitHub
parent 6ea91b2dde
commit 788b435f9b
17 changed files with 234 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
server/apps/immich/src/api-v1/asset/asset-repository.ts
View File

@@ -29,6 +29,8 @@ export interface IAssetRepository {
livePhotoAssetEntity?: AssetEntity,
): Promise<AssetEntity>;
update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
getAll(): Promise<AssetEntity[]>;
getAllVideos(): Promise<AssetEntity[]>;
getAllByUserId(userId: string, dto: AssetSearchDto): Promise<AssetEntity[]>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getById(assetId: string): Promise<AssetEntity>;
@@ -61,6 +63,22 @@ export class AssetRepository implements IAssetRepository {
@Inject(ITagRepository) private _tagRepository: ITagRepository,
) {}
async getAllVideos(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: { type: AssetType.VIDEO },
});
}
async getAll(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: { isVisible: true },
relations: {
exifInfo: true,
smartInfo: true,
},
});
}
async getAssetWithNoSmartInfo(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')

2
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@@ -123,6 +123,8 @@ describe('AssetService', () => {
assetRepositoryMock = {
create: jest.fn(),
update: jest.fn(),
getAll: jest.fn(),
getAllVideos: jest.fn(),
getAllByUserId: jest.fn(),
getAllByDeviceId: jest.fn(),
getAssetCountByTimeBucket: jest.fn(),

6
server/apps/immich/src/api-v1/job/dto/job-command.dto.ts
View File

@@ -1,5 +1,5 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsIn, IsNotEmpty } from 'class-validator';
import { IsBoolean, IsIn, IsNotEmpty, IsOptional } from 'class-validator';
export class JobCommandDto {
@IsNotEmpty()
@@ -9,4 +9,8 @@ export class JobCommandDto {
enumName: 'JobCommand',
})
command!: string;
@IsOptional()
@IsBoolean()
includeAllAssets!: boolean;
}

8
server/apps/immich/src/api-v1/job/job.controller.ts
View File

@@ -21,12 +21,12 @@ export class JobController {
@Put('/:jobId')
async sendJobCommand(
@Param(ValidationPipe) params: GetJobDto,
@Body(ValidationPipe) body: JobCommandDto,
@Body(ValidationPipe) dto: JobCommandDto,
): Promise<number> {
if (body.command === 'start') {
return await this.jobService.start(params.jobId);
if (dto.command === 'start') {
return await this.jobService.start(params.jobId, dto.includeAllAssets);
}
if (body.command === 'stop') {
if (dto.command === 'stop') {
return await this.jobService.stop(params.jobId);
}
return 0;

43
server/apps/immich/src/api-v1/job/job.service.ts
View File

@@ -5,7 +5,7 @@ import { IAssetRepository } from '../asset/asset-repository';
import { AssetType } from '@app/infra';
import { JobId } from './dto/get-job.dto';
import { MACHINE_LEARNING_ENABLED } from '@app/common';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
const jobIds = Object.values(JobId) as JobId[];
@Injectable()
@@ -19,8 +19,8 @@ export class JobService {
}
}
start(jobId: JobId): Promise<number> {
return this.run(this.asQueueName(jobId));
start(jobId: JobId, includeAllAssets: boolean): Promise<number> {
return this.run(this.asQueueName(jobId), includeAllAssets);
}
async stop(jobId: JobId): Promise<number> {
@@ -36,7 +36,7 @@ export class JobService {
return response;
}
private async run(name: QueueName): Promise<number> {
private async run(name: QueueName, includeAllAssets: boolean): Promise<number> {
const isActive = await this.jobRepository.isActive(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
@@ -44,7 +44,9 @@ export class JobService {
switch (name) {
case QueueName.VIDEO_CONVERSION: {
const assets = await this._assetRepository.getAssetWithNoEncodedVideo();
const assets = includeAllAssets
? await this._assetRepository.getAllVideos()
: await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
@@ -61,7 +63,10 @@ export class JobService {
throw new BadRequestException('Machine learning is not enabled.');
}
const assets = await this._assetRepository.getAssetWithNoSmartInfo();
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } });
@@ -70,19 +75,37 @@ export class JobService {
}
case QueueName.METADATA_EXTRACTION: {
const assets = await this._assetRepository.getAssetWithNoEXIF();
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoEXIF();
for (const asset of assets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
await this.jobRepository.add({
name: JobName.EXTRACT_VIDEO_METADATA,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
} else {
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
await this.jobRepository.add({
name: JobName.EXIF_EXTRACTION,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
}
}
return assets.length;
}
case QueueName.THUMBNAIL_GENERATION: {
const assets = await this._assetRepository.getAssetWithNoThumbnail();
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}

90
server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts
View File

@@ -1,9 +1,8 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { IsNull, Not, Repository } from 'typeorm';
import { AssetEntity, AssetType, ExifEntity, UserEntity } from '@app/infra';
import { ConfigService } from '@nestjs/config';
import { UserEntity } from '@app/infra';
import { userUtils } from '@app/common';
import { IJobRepository, JobName } from '@app/domain';
@@ -13,93 +12,8 @@ export class ScheduleTasksService {
@InjectRepository(UserEntity)
private userRepository: Repository<UserEntity>,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectRepository(ExifEntity)
private exifRepository: Repository<ExifEntity>,
@Inject(IJobRepository) private jobRepository: IJobRepository,
private configService: ConfigService,
) {}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async webpConversion() {
const assets = await this.assetRepository.find({
where: {
webpPath: '',
},
});
if (assets.length == 0) {
Logger.log('All assets has webp file - aborting task', 'CronjobWebpGenerator');
return;
}
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
}
}
@Cron(CronExpression.EVERY_DAY_AT_1AM)
async videoConversion() {
const assets = await this.assetRepository.find({
where: {
type: AssetType.VIDEO,
mimeType: 'video/quicktime',
encodedVideoPath: '',
},
order: {
createdAt: 'DESC',
},
});
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
}
@Cron(CronExpression.EVERY_DAY_AT_2AM)
async reverseGeocoding() {
const isGeocodingEnabled = this.configService.get('DISABLE_REVERSE_GEOCODING') !== 'true';
if (isGeocodingEnabled) {
const exifInfo = await this.exifRepository.find({
where: {
city: IsNull(),
longitude: Not(IsNull()),
latitude: Not(IsNull()),
},
});
for (const exif of exifInfo) {
await this.jobRepository.add({
name: JobName.REVERSE_GEOCODING,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
data: { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! },
});
}
}
}
@Cron(CronExpression.EVERY_DAY_AT_3AM)
async extractExif() {
const exifAssets = await this.assetRepository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'ei')
.where('ei."assetId" IS NULL')
.getMany();
for (const asset of exifAssets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
} else {
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
}
}
}
@Cron(CronExpression.EVERY_DAY_AT_11PM)
async deleteUserAndRelatedAssets() {
const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } });

5
server/apps/immich/src/utils/file-name.util.ts Normal file
View File

@@ -0,0 +1,5 @@
import { basename, extname } from 'node:path';
export function getFileNameWithoutExtension(path: string): string {
return basename(path, extname(path));
}

4
server/apps/microservices/src/processors/metadata-extraction.processor.ts
View File

@@ -216,7 +216,7 @@ export class MetadataExtractionProcessor {
}
}
await this.exifRepository.save(newExif);
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
} catch (error: any) {
this.logger.error(`Error extracting EXIF ${error}`, error?.stack);
}
@@ -327,7 +327,7 @@ export class MetadataExtractionProcessor {
}
}
await this.exifRepository.save(newExif);
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
} catch (err) {
// do nothing

15
server/apps/microservices/src/processors/video-transcode.processor.ts
View File

@@ -11,6 +11,7 @@ import { Repository } from 'typeorm';
@Processor(QueueName.VIDEO_CONVERSION)
export class VideoTranscodeProcessor {
readonly logger = new Logger(VideoTranscodeProcessor.name);
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@@ -20,7 +21,6 @@ export class VideoTranscodeProcessor {
@Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
async videoConversion(job: Job<IVideoConversionProcessor>) {
const { asset } = job.data;
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
@@ -30,17 +30,14 @@ export class VideoTranscodeProcessor {
const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`;
if (!asset.encodedVideoPath) {
// Put the processing into its own async function to prevent the job exist right away
await this.runVideoEncode(asset, savedEncodedPath);
}
await this.runVideoEncode(asset, savedEncodedPath);
}
async runFFProbePipeline(asset: AssetEntity): Promise<FfprobeData> {
return new Promise((resolve, reject) => {
ffmpeg.ffprobe(asset.originalPath, (err, data) => {
if (err || !data) {
Logger.error(`Cannot probe video ${err}`, 'mp4Conversion');
this.logger.error(`Cannot probe video ${err}`, 'runFFProbePipeline');
reject(err);
}
@@ -88,14 +85,14 @@ export class VideoTranscodeProcessor {
])
.output(savedEncodedPath)
.on('start', () => {
Logger.log('Start Converting Video', 'mp4Conversion');
this.logger.log('Start Converting Video');
})
.on('error', (error) => {
Logger.error(`Cannot Convert Video ${error}`, 'mp4Conversion');
this.logger.error(`Cannot Convert Video ${error}`);
reject();
})
.on('end', async () => {
Logger.log(`Converting Success ${asset.id}`, 'mp4Conversion');
this.logger.log(`Converting Success ${asset.id}`);
await this.assetRepository.update({ id: asset.id }, { encodedVideoPath: savedEncodedPath });
resolve();
})