feat: support iOS LivePhoto backup (#950)

This commit is contained in:
Alex
2022-11-18 23:12:54 -06:00
committed by GitHub
parent 83e2cabbcc
commit 8bc64be77b
30 changed files with 678 additions and 243 deletions

View File

@@ -21,7 +21,9 @@ export interface IAssetRepository {
ownerId: string,
originalPath: string,
mimeType: string,
isVisible: boolean,
checksum?: Buffer,
livePhotoAssetEntity?: AssetEntity,
): Promise<AssetEntity>;
update(asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]>;
@@ -58,6 +60,7 @@ export class AssetRepository implements IAssetRepository {
.leftJoinAndSelect('asset.smartInfo', 'si')
.where('asset.resizePath IS NOT NULL')
.andWhere('si.id IS NULL')
.andWhere('asset.isVisible = true')
.getMany();
}
@@ -65,6 +68,7 @@ export class AssetRepository implements IAssetRepository {
return await this.assetRepository
.createQueryBuilder('asset')
.where('asset.resizePath IS NULL')
.andWhere('asset.isVisible = true')
.orWhere('asset.resizePath = :resizePath', { resizePath: '' })
.orWhere('asset.webpPath IS NULL')
.orWhere('asset.webpPath = :webpPath', { webpPath: '' })
@@ -76,6 +80,7 @@ export class AssetRepository implements IAssetRepository {
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'ei')
.where('ei."assetId" IS NULL')
.andWhere('asset.isVisible = true')
.getMany();
}
@@ -86,6 +91,7 @@ export class AssetRepository implements IAssetRepository {
.select(`COUNT(asset.id)`, 'count')
.addSelect(`asset.type`, 'type')
.where('"userId" = :userId', { userId: userId })
.andWhere('asset.isVisible = true')
.groupBy('asset.type')
.getRawMany();
@@ -120,6 +126,7 @@ export class AssetRepository implements IAssetRepository {
buckets: [...getAssetByTimeBucketDto.timeBucket],
})
.andWhere('asset.resizePath is not NULL')
.andWhere('asset.isVisible = true')
.orderBy('asset.createdAt', 'DESC')
.getMany();
}
@@ -134,6 +141,7 @@ export class AssetRepository implements IAssetRepository {
.addSelect(`date_trunc('month', "createdAt")`, 'timeBucket')
.where('"userId" = :userId', { userId: userId })
.andWhere('asset.resizePath is not NULL')
.andWhere('asset.isVisible = true')
.groupBy(`date_trunc('month', "createdAt")`)
.orderBy(`date_trunc('month', "createdAt")`, 'DESC')
.getRawMany();
@@ -144,6 +152,7 @@ export class AssetRepository implements IAssetRepository {
.addSelect(`date_trunc('day', "createdAt")`, 'timeBucket')
.where('"userId" = :userId', { userId: userId })
.andWhere('asset.resizePath is not NULL')
.andWhere('asset.isVisible = true')
.groupBy(`date_trunc('day', "createdAt")`)
.orderBy(`date_trunc('day', "createdAt")`, 'DESC')
.getRawMany();
@@ -156,6 +165,7 @@ export class AssetRepository implements IAssetRepository {
return await this.assetRepository
.createQueryBuilder('asset')
.where('asset.userId = :userId', { userId: userId })
.andWhere('asset.isVisible = true')
.leftJoin('asset.exifInfo', 'ei')
.leftJoin('asset.smartInfo', 'si')
.select('si.tags', 'tags')
@@ -179,6 +189,7 @@ export class AssetRepository implements IAssetRepository {
FROM assets a
LEFT JOIN smart_info si ON a.id = si."assetId"
WHERE a."userId" = $1
AND a."isVisible" = true
AND si.objects IS NOT NULL
`,
[userId],
@@ -192,6 +203,7 @@ export class AssetRepository implements IAssetRepository {
FROM assets a
LEFT JOIN exif e ON a.id = e."assetId"
WHERE a."userId" = $1
AND a."isVisible" = true
AND e.city IS NOT NULL
AND a.type = 'IMAGE';
`,
@@ -222,6 +234,7 @@ export class AssetRepository implements IAssetRepository {
.createQueryBuilder('asset')
.where('asset.userId = :userId', { userId: userId })
.andWhere('asset.resizePath is not NULL')
.andWhere('asset.isVisible = true')
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.skip(skip || 0)
.orderBy('asset.createdAt', 'DESC');
@@ -242,13 +255,15 @@ export class AssetRepository implements IAssetRepository {
ownerId: string,
originalPath: string,
mimeType: string,
isVisible: boolean,
checksum?: Buffer,
livePhotoAssetEntity?: AssetEntity,
): Promise<AssetEntity> {
const asset = new AssetEntity();
asset.deviceAssetId = createAssetDto.deviceAssetId;
asset.userId = ownerId;
asset.deviceId = createAssetDto.deviceId;
asset.type = createAssetDto.assetType || AssetType.OTHER;
asset.type = !isVisible ? AssetType.VIDEO : createAssetDto.assetType || AssetType.OTHER; // If an asset is not visible, it is a LivePhotos video portion, therefore we can confidently assign the type as VIDEO here
asset.originalPath = originalPath;
asset.createdAt = createAssetDto.createdAt;
asset.modifiedAt = createAssetDto.modifiedAt;
@@ -256,6 +271,8 @@ export class AssetRepository implements IAssetRepository {
asset.mimeType = mimeType;
asset.duration = createAssetDto.duration || null;
asset.checksum = checksum || null;
asset.isVisible = isVisible;
asset.livePhotoVideoId = livePhotoAssetEntity ? livePhotoAssetEntity.id : null;
const createdAsset = await this.assetRepository.save(asset);
@@ -286,6 +303,7 @@ export class AssetRepository implements IAssetRepository {
where: {
userId: userId,
deviceId: deviceId,
isVisible: true,
},
select: ['deviceAssetId'],
});

View File

@@ -10,16 +10,14 @@ import {
Response,
Headers,
Delete,
Logger,
HttpCode,
BadRequestException,
UploadedFile,
Header,
Put,
UploadedFiles,
} from '@nestjs/common';
import { Authenticated } from '../../decorators/authenticated.decorator';
import { AssetService } from './asset.service';
import { FileInterceptor } from '@nestjs/platform-express';
import { FileFieldsInterceptor } from '@nestjs/platform-express';
import { assetUploadOption } from '../../config/asset-upload.config';
import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
import { ServeFileDto } from './dto/serve-file.dto';
@@ -27,12 +25,6 @@ import { Response as Res } from 'express';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { DeleteAssetDto } from './dto/delete-asset.dto';
import { SearchAssetDto } from './dto/search-asset.dto';
import { CommunicationGateway } from '../communication/communication.gateway';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { IAssetUploadedJob } from '@app/job/index';
import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
import { assetUploadedProcessorName } from '@app/job/constants/job-name.constant';
import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
import { ApiBearerAuth, ApiBody, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
@@ -47,7 +39,6 @@ import { GetAssetThumbnailDto } from './dto/get-asset-thumbnail.dto';
import { AssetCountByTimeBucketResponseDto } from './response-dto/asset-count-by-time-group-response.dto';
import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
import { QueryFailedError } from 'typeorm';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
@@ -64,17 +55,18 @@ import {
@ApiTags('Asset')
@Controller('asset')
export class AssetController {
constructor(
private wsCommunicateionGateway: CommunicationGateway,
private assetService: AssetService,
private backgroundTaskService: BackgroundTaskService,
@InjectQueue(QueueNameEnum.ASSET_UPLOADED)
private assetUploadedQueue: Queue<IAssetUploadedJob>,
) {}
constructor(private assetService: AssetService, private backgroundTaskService: BackgroundTaskService) {}
@Post('upload')
@UseInterceptors(FileInterceptor('assetData', assetUploadOption))
@UseInterceptors(
FileFieldsInterceptor(
[
{ name: 'assetData', maxCount: 1 },
{ name: 'livePhotoData', maxCount: 1 },
],
assetUploadOption,
),
)
@ApiConsumes('multipart/form-data')
@ApiBody({
description: 'Asset Upload Information',
@@ -82,53 +74,14 @@ export class AssetController {
})
async uploadFile(
@GetAuthUser() authUser: AuthUserDto,
@UploadedFile() file: Express.Multer.File,
@Body(ValidationPipe) assetInfo: CreateAssetDto,
@UploadedFiles() files: { assetData: Express.Multer.File[]; livePhotoData?: Express.Multer.File[] },
@Body(ValidationPipe) createAssetDto: CreateAssetDto,
@Response({ passthrough: true }) res: Res,
): Promise<AssetFileUploadResponseDto> {
const checksum = await this.assetService.calculateChecksum(file.path);
const originalAssetData = files.assetData[0];
const livePhotoAssetData = files.livePhotoData?.[0];
try {
const savedAsset = await this.assetService.createUserAsset(
authUser,
assetInfo,
file.path,
file.mimetype,
checksum,
);
if (!savedAsset) {
await this.backgroundTaskService.deleteFileOnDisk([
{
originalPath: file.path,
} as any,
]); // simulate asset to make use of delete queue (or use fs.unlink instead)
throw new BadRequestException('Asset not created');
}
await this.assetUploadedQueue.add(
assetUploadedProcessorName,
{ asset: savedAsset, fileName: file.originalname },
{ jobId: savedAsset.id },
);
return new AssetFileUploadResponseDto(savedAsset.id);
} catch (err) {
await this.backgroundTaskService.deleteFileOnDisk([
{
originalPath: file.path,
} as any,
]); // simulate asset to make use of delete queue (or use fs.unlink instead)
if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') {
const existedAsset = await this.assetService.getAssetByChecksum(authUser.id, checksum);
res.status(200); // normal POST is 201. we use 200 to indicate the asset already exists
return new AssetFileUploadResponseDto(existedAsset.id);
}
Logger.error(`Error uploading file ${err}`);
throw new BadRequestException(`Error uploading file`, `${err}`);
}
return this.assetService.handleUploadedAsset(authUser, createAssetDto, res, originalAssetData, livePhotoAssetData);
}
@Get('/download/:assetId')
@@ -270,6 +223,14 @@ export class AssetController {
continue;
}
deleteAssetList.push(assets);
if (assets.livePhotoVideoId) {
const livePhotoVideo = await this.assetService.getAssetById(authUser, assets.livePhotoVideoId);
if (livePhotoVideo) {
deleteAssetList.push(livePhotoVideo);
assetIds.ids = [...assetIds.ids, livePhotoVideo.id];
}
}
}
const result = await this.assetService.deleteAssetById(authUser, assetIds);

View File

@@ -25,6 +25,14 @@ import { DownloadModule } from '../../modules/download/download.module';
removeOnFail: false,
},
}),
BullModule.registerQueue({
name: QueueNameEnum.VIDEO_CONVERSION,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
removeOnFail: false,
},
}),
],
controllers: [AssetController],
providers: [

View File

@@ -8,13 +8,18 @@ import { AssetCountByTimeBucket } from './response-dto/asset-count-by-time-group
import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { DownloadService } from '../../modules/download/download.service';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { IAssetUploadedJob, IVideoTranscodeJob } from '@app/job';
import { Queue } from 'bull';
describe('AssetService', () => {
let sui: AssetService;
let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
let assetRepositoryMock: jest.Mocked<IAssetRepository>;
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
let backgroundTaskServiceMock: jest.Mocked<BackgroundTaskService>;
let assetUploadedQueueMock: jest.Mocked<Queue<IAssetUploadedJob>>;
let videoConversionQueueMock: jest.Mocked<Queue<IVideoTranscodeJob>>;
const authUser: AuthUserDto = Object.freeze({
id: 'user_id_1',
email: 'auth@test.com',
@@ -123,7 +128,14 @@ describe('AssetService', () => {
downloadArchive: jest.fn(),
};
sui = new AssetService(assetRepositoryMock, a, downloadServiceMock as DownloadService);
sui = new AssetService(
assetRepositoryMock,
a,
backgroundTaskServiceMock,
assetUploadedQueueMock,
videoConversionQueueMock,
downloadServiceMock as DownloadService,
);
});
// Currently failing due to calculate checksum from a file
@@ -141,6 +153,7 @@ describe('AssetService', () => {
originalPath,
mimeType,
Buffer.from('0x5041E6328F7DF8AFF650BEDAED9251897D9A6241', 'hex'),
true,
);
expect(result.userId).toEqual(authUser.id);

View File

@@ -10,8 +10,8 @@ import {
StreamableFile,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { createHash } from 'node:crypto';
import { Repository } from 'typeorm';
import { createHash, randomUUID } from 'node:crypto';
import { QueryFailedError, Repository } from 'typeorm';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
import { constants, createReadStream, ReadStream, stat } from 'fs';
@@ -41,6 +41,17 @@ import { timeUtils } from '@app/common/utils';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import {
assetUploadedProcessorName,
IAssetUploadedJob,
IVideoTranscodeJob,
mp4ConversionProcessorName,
QueueNameEnum,
} from '@app/job';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { DownloadService } from '../../modules/download/download.service';
import { DownloadDto } from './dto/download-library.dto';
@@ -55,15 +66,116 @@ export class AssetService {
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
private backgroundTaskService: BackgroundTaskService,
@InjectQueue(QueueNameEnum.ASSET_UPLOADED)
private assetUploadedQueue: Queue<IAssetUploadedJob>,
@InjectQueue(QueueNameEnum.VIDEO_CONVERSION)
private videoConversionQueue: Queue<IVideoTranscodeJob>,
private downloadService: DownloadService,
) {}
public async handleUploadedAsset(
authUser: AuthUserDto,
createAssetDto: CreateAssetDto,
res: Res,
originalAssetData: Express.Multer.File,
livePhotoAssetData?: Express.Multer.File,
) {
const checksum = await this.calculateChecksum(originalAssetData.path);
const isLivePhoto = livePhotoAssetData !== undefined;
let livePhotoAssetEntity: AssetEntity | undefined;
try {
if (isLivePhoto) {
const livePhotoChecksum = await this.calculateChecksum(livePhotoAssetData.path);
livePhotoAssetEntity = await this.createUserAsset(
authUser,
createAssetDto,
livePhotoAssetData.path,
livePhotoAssetData.mimetype,
livePhotoChecksum,
false,
);
if (!livePhotoAssetEntity) {
await this.backgroundTaskService.deleteFileOnDisk([
{
originalPath: livePhotoAssetData.path,
} as any,
]);
throw new BadRequestException('Asset not created');
}
await this.videoConversionQueue.add(
mp4ConversionProcessorName,
{ asset: livePhotoAssetEntity },
{ jobId: randomUUID() },
);
}
const assetEntity = await this.createUserAsset(
authUser,
createAssetDto,
originalAssetData.path,
originalAssetData.mimetype,
checksum,
true,
livePhotoAssetEntity,
);
if (!assetEntity) {
await this.backgroundTaskService.deleteFileOnDisk([
{
originalPath: originalAssetData.path,
} as any,
]);
throw new BadRequestException('Asset not created');
}
await this.assetUploadedQueue.add(
assetUploadedProcessorName,
{ asset: assetEntity, fileName: originalAssetData.originalname },
{ jobId: assetEntity.id },
);
return new AssetFileUploadResponseDto(assetEntity.id);
} catch (err) {
await this.backgroundTaskService.deleteFileOnDisk([
{
originalPath: originalAssetData.path,
} as any,
]); // simulate asset to make use of delete queue (or use fs.unlink instead)
if (isLivePhoto) {
await this.backgroundTaskService.deleteFileOnDisk([
{
originalPath: livePhotoAssetData.path,
} as any,
]);
}
if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') {
const existedAsset = await this.getAssetByChecksum(authUser.id, checksum);
res.status(200); // normal POST is 201. we use 200 to indicate the asset already exists
return new AssetFileUploadResponseDto(existedAsset.id);
}
Logger.error(`Error uploading file ${err}`);
throw new BadRequestException(`Error uploading file`, `${err}`);
}
}
public async createUserAsset(
authUser: AuthUserDto,
createAssetDto: CreateAssetDto,
originalPath: string,
mimeType: string,
checksum: Buffer,
isVisible: boolean,
livePhotoAssetEntity?: AssetEntity,
): Promise<AssetEntity> {
// Check valid time.
const createdAt = createAssetDto.createdAt;
@@ -82,7 +194,9 @@ export class AssetService {
authUser.id,
originalPath,
mimeType,
isVisible,
checksum,
livePhotoAssetEntity,
);
return assetEntity;

View File

@@ -22,6 +22,7 @@ export class AssetResponseDto {
encodedVideoPath!: string | null;
exifInfo?: ExifResponseDto;
smartInfo?: SmartInfoResponseDto;
livePhotoVideoId!: string | null;
}
export function mapAsset(entity: AssetEntity): AssetResponseDto {
@@ -42,5 +43,6 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto {
duration: entity.duration ?? '0:00:00.00000',
exifInfo: entity.exifInfo ? mapExif(entity.exifInfo) : undefined,
smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined,
livePhotoVideoId: entity.livePhotoVideoId,
};
}

View File

@@ -54,7 +54,12 @@ function filename(req: Request, file: Express.Multer.File, cb: any) {
}
const fileNameUUID = randomUUID();
if (file.fieldname === 'livePhotoData') {
const livePhotoFileName = `${fileNameUUID}.mov`;
return cb(null, sanitize(livePhotoFileName));
}
const fileName = `${fileNameUUID}${req.body['fileExtension'].toLowerCase()}`;
const sanitizedFileName = sanitize(fileName);
cb(null, sanitizedFileName);
return cb(null, sanitize(fileName));
}

View File

@@ -20,7 +20,7 @@ export class VideoTranscodeProcessor {
private immichConfigService: ImmichConfigService,
) {}
@Process({ name: mp4ConversionProcessorName, concurrency: 1 })
@Process({ name: mp4ConversionProcessorName, concurrency: 2 })
async mp4Conversion(job: Job<IMp4ConversionProcessor>) {
const { asset } = job.data;

File diff suppressed because one or more lines are too long

View File

@@ -51,6 +51,12 @@ export class AssetEntity {
@Column({ type: 'varchar', nullable: true })
duration!: string | null;
@Column({ type: 'boolean', default: true })
isVisible!: boolean;
@Column({ type: 'uuid', nullable: true })
livePhotoVideoId!: string | null;
@OneToOne(() => ExifEntity, (exifEntity) => exifEntity.asset)
exifInfo?: ExifEntity;

View File

@@ -0,0 +1,16 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddLivePhotosRelatedColumnToAssetTable1668383120461 implements MigrationInterface {
name = 'AddLivePhotosRelatedColumnToAssetTable1668383120461'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "assets" ADD "isVisible" boolean NOT NULL DEFAULT true`);
await queryRunner.query(`ALTER TABLE "assets" ADD "livePhotoVideoId" uuid`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "livePhotoVideoId"`);
await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "isVisible"`);
}
}