KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(server) Add job for storage migration (#1117)

Browse Source
This commit is contained in:
Alex
2022-12-19 12:13:10 -06:00
committed by GitHub
parent 8998a79ff9
commit de69d0031e
33 changed files with 398 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
server/apps/microservices/src/microservices.module.ts
View File

@@ -1,10 +1,10 @@
import { immichAppConfig } from '@app/common/config';
import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config';
import { DatabaseModule } from '@app/database';
import { AssetEntity } from '@app/database/entities/asset.entity';
import { ExifEntity } from '@app/database/entities/exif.entity';
import { SmartInfoEntity } from '@app/database/entities/smart-info.entity';
import { UserEntity } from '@app/database/entities/user.entity';
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
import { StorageModule } from '@app/storage';
import { BullModule } from '@nestjs/bull';
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
@@ -16,9 +16,11 @@ import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
import { GenerateChecksumProcessor } from './processors/generate-checksum.processor';
import { MachineLearningProcessor } from './processors/machine-learning.processor';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
import { StorageMigrationProcessor } from './processors/storage-migration.processor';
import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
import { UserDeletionProcessor } from './processors/user-deletion.processor';
import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
@Module({
imports: [
@@ -26,76 +28,9 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
DatabaseModule,
ImmichConfigModule,
TypeOrmModule.forFeature([UserEntity, ExifEntity, AssetEntity, SmartInfoEntity]),
BullModule.forRootAsync({
useFactory: async () => ({
prefix: 'immich_bull',
redis: {
host: process.env.REDIS_HOSTNAME || 'immich_redis',
port: parseInt(process.env.REDIS_PORT || '6379'),
db: parseInt(process.env.REDIS_DBINDEX || '0'),
password: process.env.REDIS_PASSWORD || undefined,
path: process.env.REDIS_SOCKET || undefined,
},
}),
}),
BullModule.registerQueue(
{
name: QueueNameEnum.USER_DELETION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.THUMBNAIL_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.ASSET_UPLOADED,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.METADATA_EXTRACTION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.VIDEO_CONVERSION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.CHECKSUM_GENERATION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
{
name: QueueNameEnum.MACHINE_LEARNING,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
},
),
StorageModule,
BullModule.forRootAsync(immichBullAsyncConfig),
BullModule.registerQueue(...immichSharedQueues),
CommunicationModule,
],
controllers: [],
@@ -108,7 +43,8 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
GenerateChecksumProcessor,
MachineLearningProcessor,
UserDeletionProcessor,
StorageMigrationProcessor,
],
exports: [],
exports: [BullModule],
})
export class MicroservicesModule {}

61
server/apps/microservices/src/processors/storage-migration.processor.ts Normal file
View File

@@ -0,0 +1,61 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { AssetEntity } from '@app/database/entities/asset.entity';
import { ImmichConfigService } from '@app/immich-config';
import { QueueNameEnum, templateMigrationProcessorName, updateTemplateProcessorName } from '@app/job';
import { StorageService } from '@app/storage';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
@Processor(QueueNameEnum.STORAGE_MIGRATION)
export class StorageMigrationProcessor {
readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
constructor(
private storageService: StorageService,
private immichConfigService: ImmichConfigService,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
) {}
/**
* Migration process when a new user set a new storage template.
* @param job
*/
@Process({ name: templateMigrationProcessorName, concurrency: 100 })
async templateMigration() {
console.time('migrating-time');
const assets = await this.assetRepository.find({
relations: ['exifInfo'],
});
const livePhotoMap: Record<string, AssetEntity> = {};
for (const asset of assets) {
if (asset.livePhotoVideoId) {
livePhotoMap[asset.livePhotoVideoId] = asset;
}
}
for (const asset of assets) {
const livePhotoParentAsset = livePhotoMap[asset.id];
const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
await this.storageService.moveAsset(asset, filename);
}
await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
console.timeEnd('migrating-time');
}
/**
* Update config when a new storage template is set.
* This is to ensure the synchronization between processes.
* @param job
*/
@Process({ name: updateTemplateProcessorName, concurrency: 1 })
async updateTemplate() {
await this.immichConfigService.refreshConfig();
}
}

27
server/apps/microservices/src/processors/thumbnail.processor.ts
View File

@@ -1,5 +1,4 @@
import { APP_UPLOAD_LOCATION } from '@app/common';
import { ImmichLogLevel } from '@app/common/constants/log-level.constant';
import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
import {
WebpGeneratorProcessor,
@@ -11,7 +10,6 @@ import {
} from '@app/job';
import { InjectQueue, Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { mapAsset } from 'apps/immich/src/api-v1/asset/response-dto/asset-response.dto';
import { Job, Queue } from 'bull';
@@ -27,7 +25,7 @@ import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interf
@Processor(QueueNameEnum.THUMBNAIL_GENERATION)
export class ThumbnailGeneratorProcessor {
private logLevel: ImmichLogLevel;
readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name);
constructor(
@InjectRepository(AssetEntity)
@@ -40,12 +38,7 @@ export class ThumbnailGeneratorProcessor {
@InjectQueue(QueueNameEnum.MACHINE_LEARNING)
private machineLearningQueue: Queue<IMachineLearningJob>,
private configService: ConfigService,
) {
this.logLevel = this.configService.get('LOG_LEVEL') || ImmichLogLevel.SIMPLE;
// TODO - Add observable paterrn to listen to the config change
}
) {}
@Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 })
async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
@@ -70,12 +63,8 @@ export class ThumbnailGeneratorProcessor {
.rotate()
.toFile(jpegThumbnailPath);
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
} catch (error) {
Logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id);
if (this.logLevel == ImmichLogLevel.VERBOSE) {
console.trace('Failed to generate jpeg thumbnail for asset', error);
}
} catch (error: any) {
this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
}
// Update resize path to send to generate webp queue
@@ -140,12 +129,8 @@ export class ThumbnailGeneratorProcessor {
try {
await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath);
await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });
} catch (error) {
Logger.error('Failed to generate webp thumbnail for asset: ' + asset.id);
if (this.logLevel == ImmichLogLevel.VERBOSE) {
console.trace('Failed to generate webp thumbnail for asset', error);
}
} catch (error: any) {
this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
}
}
}