feat(server,web): libraries (#3124)

* feat: libraries

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors
2023-09-20 13:16:33 +02:00
committed by GitHub
parent 816db700e1
commit acdc66413c
143 changed files with 10941 additions and 386 deletions

View File

@@ -21,7 +21,13 @@ export enum Permission {
ARCHIVE_READ = 'archive.read',
TIMELINE_READ = 'timeline.read',
TIMELINE_DOWNLOAD = 'timeline.download',
LIBRARY_CREATE = 'library.create',
LIBRARY_READ = 'library.read',
LIBRARY_UPDATE = 'library.update',
LIBRARY_DELETE = 'library.delete',
LIBRARY_DOWNLOAD = 'library.download',
PERSON_READ = 'person.read',
@@ -165,12 +171,24 @@ export class AccessCore {
case Permission.ARCHIVE_READ:
return authUser.id === id;
case Permission.LIBRARY_READ:
return authUser.id === id || (await this.repository.library.hasPartnerAccess(authUser.id, id));
case Permission.TIMELINE_READ:
return authUser.id === id || (await this.repository.timeline.hasPartnerAccess(authUser.id, id));
case Permission.LIBRARY_DOWNLOAD:
case Permission.TIMELINE_DOWNLOAD:
return authUser.id === id;
case Permission.LIBRARY_READ:
return (
(await this.repository.library.hasOwnerAccess(authUser.id, id)) ||
(await this.repository.library.hasPartnerAccess(authUser.id, id))
);
case Permission.LIBRARY_UPDATE:
return this.repository.library.hasOwnerAccess(authUser.id, id);
case Permission.LIBRARY_DELETE:
return this.repository.library.hasOwnerAccess(authUser.id, id);
case Permission.PERSON_READ:
return this.repository.person.hasOwnerAccess(authUser.id, id);

View File

@@ -15,6 +15,11 @@ export interface IAccessRepository {
};
library: {
hasOwnerAccess(userId: string, libraryId: string): Promise<boolean>;
hasPartnerAccess(userId: string, partnerId: string): Promise<boolean>;
};
timeline: {
hasPartnerAccess(userId: string, partnerId: string): Promise<boolean>;
};

View File

@@ -45,6 +45,7 @@ export enum WithoutProperty {
export enum WithProperty {
SIDECAR = 'sidecar',
IS_OFFLINE = 'isOffline',
}
export enum TimeBucketSize {
@@ -69,15 +70,18 @@ export interface TimeBucketItem {
export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
create(asset: Partial<AssetEntity>): Promise<AssetEntity>;
getByDate(ownerId: string, date: Date): Promise<AssetEntity[]>;
getByIds(ids: string[]): Promise<AssetEntity[]>;
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
getByUserId(pagination: PaginationOptions, userId: string): Paginated<AssetEntity>;
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
getWith(pagination: PaginationOptions, property: WithProperty): Paginated<AssetEntity>;
getWith(pagination: PaginationOptions, property: WithProperty, libraryId?: string): Paginated<AssetEntity>;
getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]>;
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
deleteAll(ownerId: string): Promise<void>;
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void>;
@@ -87,5 +91,7 @@ export interface IAssetRepository {
getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
getByTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
remove(asset: AssetEntity): Promise<AssetEntity>;
getById(assetId: string): Promise<AssetEntity>;
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
}

View File

@@ -501,6 +501,7 @@ describe(AssetService.name, () => {
});
it('should return a list of archives (userId)', async () => {
accessMock.library.hasOwnerAccess.mockResolvedValue(true);
assetMock.getByUserId.mockResolvedValue({
items: [assetStub.image, assetStub.video],
hasNextPage: false,
@@ -514,6 +515,8 @@ describe(AssetService.name, () => {
});
it('should split archives by size', async () => {
accessMock.library.hasOwnerAccess.mockResolvedValue(true);
assetMock.getByUserId.mockResolvedValue({
items: [
{ ...assetStub.image, id: 'asset-1' },

View File

@@ -162,7 +162,7 @@ export class AssetService {
if (dto.isArchived !== false) {
await this.access.requirePermission(authUser, Permission.ARCHIVE_READ, [dto.userId]);
}
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, [dto.userId]);
await this.access.requirePermission(authUser, Permission.TIMELINE_READ, [dto.userId]);
} else {
dto.userId = authUser.id;
}
@@ -187,6 +187,10 @@ export class AssetService {
throw new BadRequestException('Asset not found');
}
if (asset.isOffline) {
throw new BadRequestException('Asset is offline');
}
return this.storageRepository.createReadStream(asset.originalPath, mimeTypes.lookup(asset.originalPath));
}
@@ -268,7 +272,7 @@ export class AssetService {
if (dto.userId) {
const userId = dto.userId;
await this.access.requirePermission(authUser, Permission.LIBRARY_DOWNLOAD, userId);
await this.access.requirePermission(authUser, Permission.TIMELINE_DOWNLOAD, userId);
return usePagination(PAGINATION_SIZE, (pagination) => this.assetRepository.getByUserId(pagination, userId));
}

View File

@@ -12,6 +12,7 @@ export class AssetResponseDto {
deviceId!: string;
ownerId!: string;
owner?: UserResponseDto;
libraryId!: string;
@ApiProperty({ enumName: 'AssetTypeEnum', enum: AssetType })
type!: AssetType;
@@ -25,6 +26,9 @@ export class AssetResponseDto {
updatedAt!: Date;
isFavorite!: boolean;
isArchived!: boolean;
isOffline!: boolean;
isExternal!: boolean;
isReadOnly!: boolean;
duration!: string;
exifInfo?: ExifResponseDto;
smartInfo?: SmartInfoResponseDto;
@@ -42,6 +46,7 @@ function _map(entity: AssetEntity, withExif: boolean): AssetResponseDto {
ownerId: entity.ownerId,
owner: entity.owner ? mapUser(entity.owner) : undefined,
deviceId: entity.deviceId,
libraryId: entity.libraryId,
type: entity.type,
originalPath: entity.originalPath,
originalFileName: entity.originalFileName,
@@ -59,6 +64,9 @@ function _map(entity: AssetEntity, withExif: boolean): AssetResponseDto {
tags: entity.tags?.map(mapTag),
people: entity.faces?.map(mapFace).filter((person) => !person.isHidden),
checksum: entity.checksum.toString('base64'),
isExternal: entity.isExternal,
isOffline: entity.isOffline,
isReadOnly: entity.isReadOnly,
};
}

View File

@@ -25,7 +25,7 @@ export class AuditService {
async getDeletes(authUser: AuthUserDto, dto: AuditDeletesDto): Promise<AuditDeletesResponseDto> {
const userId = dto.userId || authUser.id;
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
await this.access.requirePermission(authUser, Permission.TIMELINE_READ, userId);
const audits = await this.repository.getAfter(dto.after, {
ownerId: userId,

View File

@@ -6,6 +6,7 @@ import {
loginResponseStub,
newCryptoRepositoryMock,
newKeyRepositoryMock,
newLibraryRepositoryMock,
newSharedLinkRepositoryMock,
newSystemConfigRepositoryMock,
newUserRepositoryMock,
@@ -20,6 +21,7 @@ import { Issuer, generators } from 'openid-client';
import { Socket } from 'socket.io';
import { IKeyRepository } from '../api-key';
import { ICryptoRepository } from '../crypto/crypto.repository';
import { ILibraryRepository } from '../library';
import { ISharedLinkRepository } from '../shared-link';
import { ISystemConfigRepository } from '../system-config';
import { IUserRepository } from '../user';
@@ -50,6 +52,7 @@ describe('AuthService', () => {
let sut: AuthService;
let cryptoMock: jest.Mocked<ICryptoRepository>;
let userMock: jest.Mocked<IUserRepository>;
let libraryMock: jest.Mocked<ILibraryRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let userTokenMock: jest.Mocked<IUserTokenRepository>;
let shareMock: jest.Mocked<ISharedLinkRepository>;
@@ -81,12 +84,13 @@ describe('AuthService', () => {
cryptoMock = newCryptoRepositoryMock();
userMock = newUserRepositoryMock();
libraryMock = newLibraryRepositoryMock();
configMock = newSystemConfigRepositoryMock();
userTokenMock = newUserTokenRepositoryMock();
shareMock = newSharedLinkRepositoryMock();
keyMock = newKeyRepositoryMock();
sut = new AuthService(cryptoMock, configMock, userMock, userTokenMock, shareMock, keyMock);
sut = new AuthService(cryptoMock, configMock, userMock, userTokenMock, libraryMock, shareMock, keyMock);
});
it('should be defined', () => {

View File

@@ -13,6 +13,7 @@ import { DateTime } from 'luxon';
import { ClientMetadata, Issuer, UserinfoResponse, custom, generators } from 'openid-client';
import { IKeyRepository } from '../api-key';
import { ICryptoRepository } from '../crypto/crypto.repository';
import { ILibraryRepository } from '../library';
import { ISharedLinkRepository } from '../shared-link';
import { ISystemConfigRepository } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
@@ -66,11 +67,12 @@ export class AuthService {
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IUserRepository) userRepository: IUserRepository,
@Inject(IUserTokenRepository) private userTokenRepository: IUserTokenRepository,
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
@Inject(ISharedLinkRepository) private sharedLinkRepository: ISharedLinkRepository,
@Inject(IKeyRepository) private keyRepository: IKeyRepository,
) {
this.configCore = new SystemConfigCore(configRepository);
this.userCore = new UserCore(userRepository, cryptoRepository);
this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
custom.setHttpOptionsDefaults({ timeout: 30000 });
}

View File

@@ -102,6 +102,7 @@ export const mimeTypes = {
video,
isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
isImage: (filename: string) => isType(filename, image),
isProfile: (filename: string) => isType(filename, profile),
isSidecar: (filename: string) => isType(filename, sidecar),
isVideo: (filename: string) => isType(filename, video),
@@ -115,4 +116,5 @@ export const mimeTypes = {
}
return AssetType.OTHER;
},
getSupportedFileExtensions: () => Object.keys(image).concat(Object.keys(video)),
};

View File

@@ -6,6 +6,7 @@ import { AuditService } from './audit';
import { AuthService } from './auth';
import { FacialRecognitionService } from './facial-recognition';
import { JobService } from './job';
import { LibraryService } from './library';
import { MediaService } from './media';
import { MetadataService } from './metadata';
import { PartnerService } from './partner';
@@ -30,6 +31,7 @@ const providers: Provider[] = [
JobService,
MediaService,
MetadataService,
LibraryService,
PersonService,
PartnerService,
SearchService,

View File

@@ -12,6 +12,7 @@ export * from './domain.module';
export * from './domain.util';
export * from './facial-recognition';
export * from './job';
export * from './library';
export * from './media';
export * from './metadata';
export * from './partner';

View File

@@ -9,6 +9,7 @@ export enum QueueName {
STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
SEARCH = 'search',
SIDECAR = 'sidecar',
LIBRARY = 'library',
}
export enum JobCommand {
@@ -53,6 +54,15 @@ export enum JobName {
RECOGNIZE_FACES = 'recognize-faces',
PERSON_CLEANUP = 'person-cleanup',
// library managment
LIBRARY_SCAN = 'library-refresh',
LIBRARY_SCAN_ASSET = 'library-refresh-asset',
LIBRARY_REMOVE_OFFLINE = 'library-remove-offline',
LIBRARY_MARK_ASSET_OFFLINE = 'library-mark-asset-offline',
LIBRARY_DELETE = 'library-delete',
LIBRARY_QUEUE_SCAN_ALL = 'library-queue-all-refresh',
LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup',
// cleanup
DELETE_FILES = 'delete-files',
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
@@ -140,4 +150,13 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,
[JobName.SIDECAR_SYNC]: QueueName.SIDECAR,
// Library managment
[JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY,
[JobName.LIBRARY_MARK_ASSET_OFFLINE]: QueueName.LIBRARY,
[JobName.LIBRARY_SCAN]: QueueName.LIBRARY,
[JobName.LIBRARY_DELETE]: QueueName.LIBRARY,
[JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY,
[JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY,
};

View File

@@ -79,4 +79,7 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
@ApiProperty({ type: JobStatusDto })
[QueueName.SIDECAR]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.LIBRARY]!: JobStatusDto;
}

View File

@@ -22,6 +22,21 @@ export interface IEntityJob extends IBaseJob {
source?: 'upload';
}
export interface IOfflineLibraryFileJob extends IEntityJob {
assetPath: string;
}
export interface ILibraryFileJob extends IEntityJob {
ownerId: string;
assetPath: string;
forceRefresh: boolean;
}
export interface ILibraryRefreshJob extends IEntityJob {
refreshModifiedFiles: boolean;
refreshAllFiles: boolean;
}
export interface IBulkEntityJob extends IBaseJob {
ids: string[];
}

View File

@@ -1,4 +1,5 @@
import { JobName, QueueName } from './job.constants';
import {
IAssetFaceJob,
IBaseJob,
@@ -6,6 +7,9 @@ import {
IDeleteFilesJob,
IEntityJob,
IFaceThumbnailJob,
ILibraryFileJob,
ILibraryRefreshJob,
IOfflineLibraryFileJob,
} from './job.interface';
export interface JobCounts {
@@ -74,6 +78,15 @@ export type JobItem =
// Asset Deletion
| { name: JobName.PERSON_CLEANUP; data?: IBaseJob }
// Library Managment
| { name: JobName.LIBRARY_SCAN_ASSET; data: ILibraryFileJob }
| { name: JobName.LIBRARY_MARK_ASSET_OFFLINE; data: IOfflineLibraryFileJob }
| { name: JobName.LIBRARY_SCAN; data: ILibraryRefreshJob }
| { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
| { name: JobName.LIBRARY_DELETE; data: IEntityJob }
| { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }
// Search
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }

View File

@@ -52,6 +52,7 @@ describe(JobService.name, () => {
[{ name: JobName.PERSON_CLEANUP }],
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
[{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }],
]);
});
});
@@ -97,6 +98,7 @@ describe(JobService.name, () => {
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
[QueueName.SIDECAR]: expectedJobStatus,
[QueueName.LIBRARY]: expectedJobStatus,
});
});
});
@@ -225,6 +227,7 @@ describe(JobService.name, () => {
[QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
[QueueName.SEARCH]: { concurrency: 10 },
[QueueName.SIDECAR]: { concurrency: 10 },
[QueueName.LIBRARY]: { concurrency: 10 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
@@ -237,6 +240,7 @@ describe(JobService.name, () => {
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.OBJECT_TAGGING, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);

View File

@@ -98,6 +98,9 @@ export class JobService {
await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } });
case QueueName.LIBRARY:
return this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force } });
default:
throw new BadRequestException(`Invalid job name: ${name}`);
}
@@ -118,7 +121,7 @@ export class JobService {
await this.onDone(item);
}
} catch (error: Error | any) {
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, data);
this.logger.error(`Unable to run job handler (${queueName}/${name}): ${error}`, error?.stack, data);
}
});
}
@@ -138,6 +141,7 @@ export class JobService {
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } });
}
/**

View File

@@ -0,0 +1,3 @@
export * from './library.dto';
export * from './library.repository';
export * from './library.service';

View File

@@ -0,0 +1,124 @@
import { LibraryEntity, LibraryType } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator';
import { ValidateUUID } from '../domain.util';
export class CreateLibraryDto {
@IsEnum(LibraryType)
@ApiProperty({ enumName: 'LibraryType', enum: LibraryType })
type!: LibraryType;
@IsString()
@IsOptional()
@IsNotEmpty()
name?: string;
@IsOptional()
@IsBoolean()
isVisible?: boolean;
@IsOptional()
@IsString({ each: true })
@IsNotEmpty({ each: true })
importPaths?: string[];
@IsOptional()
@IsString({ each: true })
@IsNotEmpty({ each: true })
exclusionPatterns?: string[];
}
export class UpdateLibraryDto {
@IsOptional()
@IsString()
@IsNotEmpty()
name?: string;
@IsOptional()
@IsBoolean()
isVisible?: boolean;
@IsOptional()
@IsString({ each: true })
@IsNotEmpty({ each: true })
importPaths?: string[];
@IsOptional()
@IsNotEmpty({ each: true })
@IsString({ each: true })
exclusionPatterns?: string[];
}
export class CrawlOptionsDto {
pathsToCrawl!: string[];
includeHidden? = false;
exclusionPatterns?: string[];
}
export class LibrarySearchDto {
@ValidateUUID({ optional: true })
userId?: string;
}
export class ScanLibraryDto {
@IsBoolean()
@IsOptional()
refreshModifiedFiles?: boolean;
@IsBoolean()
@IsOptional()
refreshAllFiles?: boolean = false;
}
export class LibraryResponseDto {
id!: string;
ownerId!: string;
name!: string;
@ApiProperty({ enumName: 'LibraryType', enum: LibraryType })
type!: LibraryType;
@ApiProperty({ type: 'integer' })
assetCount!: number;
importPaths!: string[];
exclusionPatterns!: string[];
createdAt!: Date;
updatedAt!: Date;
refreshedAt!: Date | null;
}
export class LibraryStatsResponseDto {
@ApiProperty({ type: 'integer' })
photos = 0;
@ApiProperty({ type: 'integer' })
videos = 0;
@ApiProperty({ type: 'integer' })
total = 0;
@ApiProperty({ type: 'integer', format: 'int64' })
usage = 0;
}
export function mapLibrary(entity: LibraryEntity): LibraryResponseDto {
let assetCount = 0;
if (entity.assets) {
assetCount = entity.assets.length;
}
return {
id: entity.id,
ownerId: entity.ownerId,
type: entity.type,
name: entity.name,
createdAt: entity.createdAt,
updatedAt: entity.updatedAt,
refreshedAt: entity.refreshedAt,
assetCount,
importPaths: entity.importPaths,
exclusionPatterns: entity.exclusionPatterns,
};
}

View File

@@ -0,0 +1,22 @@
import { LibraryEntity, LibraryType } from '@app/infra/entities';
import { LibraryStatsResponseDto } from './library.dto';
export const ILibraryRepository = 'ILibraryRepository';
export interface ILibraryRepository {
getCountForUser(ownerId: string): Promise<number>;
getAllByUserId(userId: string, type?: LibraryType): Promise<LibraryEntity[]>;
getAll(withDeleted?: boolean, type?: LibraryType): Promise<LibraryEntity[]>;
getAllDeleted(): Promise<LibraryEntity[]>;
get(id: string, withDeleted?: boolean): Promise<LibraryEntity | null>;
create(library: Partial<LibraryEntity>): Promise<LibraryEntity>;
delete(id: string): Promise<void>;
softDelete(id: string): Promise<void>;
getDefaultUploadLibrary(ownerId: string): Promise<LibraryEntity | null>;
getUploadLibraryCount(ownerId: string): Promise<number>;
update(library: Partial<LibraryEntity>): Promise<LibraryEntity>;
getStatistics(id: string): Promise<LibraryStatsResponseDto>;
getOnlineAssetPaths(id: string): Promise<string[]>;
getAssetIds(id: string, withDeleted?: boolean): Promise<string[]>;
existsByName(name: string, withDeleted?: boolean): Promise<boolean>;
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,468 @@
import { AssetType, LibraryType } from '@app/infra/entities';
import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
import { R_OK } from 'node:constants';
import { Stats } from 'node:fs';
import path from 'node:path';
import { basename, parse } from 'path';
import { AccessCore, IAccessRepository, Permission } from '../access';
import { IAssetRepository, WithProperty } from '../asset';
import { AuthUserDto } from '../auth';
import { usePagination } from '../domain.util';
import { ICryptoRepository } from '../crypto';
import { mimeTypes } from '../domain.constant';
import {
IBaseJob,
IEntityJob,
IJobRepository,
ILibraryFileJob,
ILibraryRefreshJob,
IOfflineLibraryFileJob,
JobName,
JOBS_ASSET_PAGINATION_SIZE,
} from '../job';
import { IStorageRepository } from '../storage';
import { IUserRepository } from '../user';
import {
CreateLibraryDto,
LibraryResponseDto,
LibraryStatsResponseDto,
mapLibrary,
ScanLibraryDto,
UpdateLibraryDto,
} from './library.dto';
import { ILibraryRepository } from './library.repository';
@Injectable()
export class LibraryService {
readonly logger = new Logger(LibraryService.name);
private access: AccessCore;
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ILibraryRepository) private repository: ILibraryRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IUserRepository) private userRepository: IUserRepository,
) {
this.access = new AccessCore(accessRepository);
}
async getStatistics(authUser: AuthUserDto, id: string): Promise<LibraryStatsResponseDto> {
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, id);
return this.repository.getStatistics(id);
}
async getCount(authUser: AuthUserDto): Promise<number> {
return this.repository.getCountForUser(authUser.id);
}
async getAllForUser(authUser: AuthUserDto): Promise<LibraryResponseDto[]> {
const libraries = await this.repository.getAllByUserId(authUser.id);
return libraries.map((library) => mapLibrary(library));
}
async get(authUser: AuthUserDto, id: string): Promise<LibraryResponseDto> {
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, id);
const library = await this.findOrFail(id);
return mapLibrary(library);
}
async handleQueueCleanup(): Promise<boolean> {
this.logger.debug('Cleaning up any pending library deletions');
const pendingDeletion = await this.repository.getAllDeleted();
for (const libraryToDelete of pendingDeletion) {
await this.jobRepository.queue({ name: JobName.LIBRARY_DELETE, data: { id: libraryToDelete.id } });
}
return true;
}
async create(authUser: AuthUserDto, dto: CreateLibraryDto): Promise<LibraryResponseDto> {
switch (dto.type) {
case LibraryType.EXTERNAL:
if (!dto.name) {
dto.name = 'New External Library';
}
break;
case LibraryType.UPLOAD:
if (!dto.name) {
dto.name = 'New Upload Library';
}
if (dto.importPaths && dto.importPaths.length > 0) {
throw new BadRequestException('Upload libraries cannot have import paths');
}
if (dto.exclusionPatterns && dto.exclusionPatterns.length > 0) {
throw new BadRequestException('Upload libraries cannot have exclusion patterns');
}
break;
}
const library = await this.repository.create({
ownerId: authUser.id,
name: dto.name,
type: dto.type,
importPaths: dto.importPaths ?? [],
exclusionPatterns: dto.exclusionPatterns ?? [],
isVisible: dto.isVisible ?? true,
});
return mapLibrary(library);
}
async update(authUser: AuthUserDto, id: string, dto: UpdateLibraryDto): Promise<LibraryResponseDto> {
await this.access.requirePermission(authUser, Permission.LIBRARY_UPDATE, id);
const library = await this.repository.update({ id, ...dto });
return mapLibrary(library);
}
async delete(authUser: AuthUserDto, id: string) {
await this.access.requirePermission(authUser, Permission.LIBRARY_DELETE, id);
const library = await this.findOrFail(id);
const uploadCount = await this.repository.getUploadLibraryCount(authUser.id);
if (library.type === LibraryType.UPLOAD && uploadCount <= 1) {
throw new BadRequestException('Cannot delete the last upload library');
}
await this.repository.softDelete(id);
await this.jobRepository.queue({ name: JobName.LIBRARY_DELETE, data: { id } });
}
async handleDeleteLibrary(job: IEntityJob): Promise<boolean> {
const library = await this.repository.get(job.id, true);
if (!library) {
return false;
}
// TODO use pagination
const assetIds = await this.repository.getAssetIds(job.id);
this.logger.debug(`Will delete ${assetIds.length} asset(s) in library ${job.id}`);
// TODO queue a job for asset deletion
await this.deleteAssets(assetIds);
this.logger.log(`Deleting library ${job.id}`);
await this.repository.delete(job.id);
return true;
}
async handleAssetRefresh(job: ILibraryFileJob) {
const assetPath = path.normalize(job.assetPath);
const user = await this.userRepository.get(job.ownerId);
if (!user?.externalPath) {
this.logger.warn('User has no external path set, cannot import asset');
return false;
}
if (!path.normalize(assetPath).match(new RegExp(`^${user.externalPath}`))) {
this.logger.error("Asset must be within the user's external path");
return false;
}
const existingAssetEntity = await this.assetRepository.getByLibraryIdAndOriginalPath(job.id, assetPath);
let stats: Stats;
try {
stats = await this.storageRepository.stat(assetPath);
} catch (error: Error | any) {
// Can't access file, probably offline
if (existingAssetEntity) {
// Mark asset as offline
this.logger.debug(`Marking asset as offline: ${assetPath}`);
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: true });
return true;
} else {
// File can't be accessed and does not already exist in db
throw new BadRequestException("Can't access file", { cause: error });
}
}
let doImport = false;
let doRefresh = false;
if (job.forceRefresh) {
doRefresh = true;
}
if (!existingAssetEntity) {
// This asset is new to us, read it from disk
this.logger.debug(`Importing new asset: ${assetPath}`);
doImport = true;
} else if (stats.mtime.toISOString() !== existingAssetEntity.fileModifiedAt.toISOString()) {
// File modification time has changed since last time we checked, re-read from disk
this.logger.debug(
`File modification time has changed, re-importing asset: ${assetPath}. Old mtime: ${existingAssetEntity.fileModifiedAt}. New mtime: ${stats.mtime}`,
);
doRefresh = true;
} else if (!job.forceRefresh && stats && !existingAssetEntity.isOffline) {
// Asset exists on disk and in db and mtime has not changed. Also, we are not forcing refresn. Therefore, do nothing
this.logger.debug(`Asset already exists in database and on disk, will not import: ${assetPath}`);
}
if (stats && existingAssetEntity?.isOffline) {
// File was previously offline but is now online
this.logger.debug(`Marking previously-offline asset as online: ${assetPath}`);
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: false });
doRefresh = true;
}
if (!doImport && !doRefresh) {
// If we don't import, exit here
return true;
}
let assetType: AssetType;
if (mimeTypes.isImage(assetPath)) {
assetType = AssetType.IMAGE;
} else if (mimeTypes.isVideo(assetPath)) {
assetType = AssetType.VIDEO;
} else {
throw new BadRequestException(`Unsupported file type ${assetPath}`);
}
// TODO: doesn't xmp replace the file extension? Will need investigation
let sidecarPath: string | null = null;
if (await this.storageRepository.checkFileExists(`${assetPath}.xmp`, R_OK)) {
sidecarPath = `${assetPath}.xmp`;
}
const deviceAssetId = `${basename(assetPath)}`.replace(/\s+/g, '');
const pathHash = this.cryptoRepository.hashSha1(`path:${assetPath}`);
let assetId;
if (doImport) {
const library = await this.repository.get(job.id, true);
if (library?.deletedAt) {
this.logger.error('Cannot import asset into deleted library');
return false;
}
// TODO: In wait of refactoring the domain asset service, this function is just manually written like this
const addedAsset = await this.assetRepository.create({
ownerId: job.ownerId,
libraryId: job.id,
checksum: pathHash,
originalPath: assetPath,
deviceAssetId: deviceAssetId,
deviceId: 'Library Import',
fileCreatedAt: stats.ctime,
fileModifiedAt: stats.mtime,
type: assetType,
originalFileName: parse(assetPath).name,
sidecarPath,
isReadOnly: true,
isExternal: true,
});
assetId = addedAsset.id;
} else if (doRefresh && existingAssetEntity) {
assetId = existingAssetEntity.id;
await this.assetRepository.updateAll([existingAssetEntity.id], {
fileCreatedAt: stats.ctime,
fileModifiedAt: stats.mtime,
});
} else {
// Not importing and not refreshing, do nothing
return true;
}
this.logger.debug(`Queuing metadata extraction for: ${assetPath}`);
await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: assetId, source: 'upload' } });
if (assetType === AssetType.VIDEO) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id: assetId } });
}
return true;
}
async queueScan(authUser: AuthUserDto, id: string, dto: ScanLibraryDto) {
await this.access.requirePermission(authUser, Permission.LIBRARY_UPDATE, id);
const library = await this.repository.get(id);
if (!library || library.type !== LibraryType.EXTERNAL) {
throw new BadRequestException('Can only refresh external libraries');
}
await this.jobRepository.queue({
name: JobName.LIBRARY_SCAN,
data: {
id,
refreshModifiedFiles: dto.refreshModifiedFiles ?? false,
refreshAllFiles: dto.refreshAllFiles ?? false,
},
});
}
async queueRemoveOffline(authUser: AuthUserDto, id: string) {
this.logger.verbose(`Removing offline files from library: ${id}`);
await this.access.requirePermission(authUser, Permission.LIBRARY_UPDATE, id);
await this.jobRepository.queue({
name: JobName.LIBRARY_REMOVE_OFFLINE,
data: {
id,
},
});
}
async handleQueueAllScan(job: IBaseJob): Promise<boolean> {
this.logger.debug(`Refreshing all external libraries: force=${job.force}`);
// Queue cleanup
await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_CLEANUP, data: {} });
// Queue all library refresh
const libraries = await this.repository.getAll(true, LibraryType.EXTERNAL);
for (const library of libraries) {
await this.jobRepository.queue({
name: JobName.LIBRARY_SCAN,
data: {
id: library.id,
refreshModifiedFiles: !job.force,
refreshAllFiles: job.force ?? false,
},
});
}
return true;
}
async handleOfflineRemoval(job: IEntityJob): Promise<boolean> {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return this.assetRepository.getWith(pagination, WithProperty.IS_OFFLINE, job.id);
});
const assetIds: string[] = [];
for await (const assets of assetPagination) {
for (const asset of assets) {
assetIds.push(asset.id);
}
}
this.logger.verbose(`Found ${assetIds.length} offline assets to remove`);
await this.deleteAssets(assetIds);
return true;
}
async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise<boolean> {
const library = await this.repository.get(job.id);
if (!library || library.type !== LibraryType.EXTERNAL) {
this.logger.warn('Can only refresh external libraries');
return false;
}
const user = await this.userRepository.get(library.ownerId);
if (!user?.externalPath) {
this.logger.warn('User has no external path set, cannot refresh library');
return false;
}
this.logger.verbose(`Refreshing library: ${job.id}`);
const crawledAssetPaths = (
await this.storageRepository.crawl({
pathsToCrawl: library.importPaths,
exclusionPatterns: library.exclusionPatterns,
})
)
.map(path.normalize)
.filter((assetPath) =>
// Filter out paths that are not within the user's external path
assetPath.match(new RegExp(`^${user.externalPath}`)),
);
this.logger.debug(`Found ${crawledAssetPaths.length} assets when crawling import paths ${library.importPaths}`);
const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]);
const offlineAssets = assetsInLibrary.filter((asset) => !crawledAssetPaths.includes(asset.originalPath));
this.logger.debug(`${offlineAssets.length} assets in library are not present on disk and will be marked offline`);
for (const offlineAsset of offlineAssets) {
const offlineJobData: IOfflineLibraryFileJob = {
id: job.id,
assetPath: offlineAsset.originalPath,
};
await this.jobRepository.queue({ name: JobName.LIBRARY_MARK_ASSET_OFFLINE, data: offlineJobData });
}
if (crawledAssetPaths.length > 0) {
let filteredPaths: string[] = [];
if (job.refreshAllFiles || job.refreshModifiedFiles) {
filteredPaths = crawledAssetPaths;
} else {
const existingPaths = await this.repository.getOnlineAssetPaths(job.id);
this.logger.debug(`Found ${existingPaths.length} existing asset(s) in library ${job.id}`);
filteredPaths = crawledAssetPaths.filter((assetPath) => !existingPaths.includes(assetPath));
this.logger.debug(`After db comparison, ${filteredPaths.length} asset(s) remain to be imported`);
}
for (const assetPath of filteredPaths) {
const libraryJobData: ILibraryFileJob = {
id: job.id,
assetPath: path.normalize(assetPath),
ownerId: library.ownerId,
forceRefresh: job.refreshAllFiles ?? false,
};
await this.jobRepository.queue({ name: JobName.LIBRARY_SCAN_ASSET, data: libraryJobData });
}
}
await this.repository.update({ id: job.id, refreshedAt: new Date() });
return true;
}
async handleOfflineAsset(job: IOfflineLibraryFileJob): Promise<boolean> {
const existingAssetEntity = await this.assetRepository.getByLibraryIdAndOriginalPath(job.id, job.assetPath);
if (existingAssetEntity) {
this.logger.verbose(`Marking asset as offline: ${job.assetPath}`);
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: true });
}
return true;
}
private async findOrFail(id: string) {
const library = await this.repository.get(id);
if (!library) {
throw new BadRequestException('Library not found');
}
return library;
}
private async deleteAssets(assetIds: string[]) {
// TODO: this should be refactored to a centralized asset deletion service
for (const assetId of assetIds) {
const asset = await this.assetRepository.getById(assetId);
this.logger.debug(`Removing asset from library: ${asset.originalPath}`);
if (asset.faces) {
await Promise.all(
asset.faces.map(({ assetId, personId }) =>
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
),
);
}
await this.assetRepository.remove(asset);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [asset.id] } });
await this.jobRepository.queue({
name: JobName.DELETE_FILES,
data: { files: [asset.webpPath, asset.resizePath, asset.encodedVideoPath, asset.sidecarPath] },
});
// TODO refactor this to use cascades
if (asset.livePhotoVideoId && !assetIds.includes(asset.livePhotoVideoId)) {
assetIds.push(asset.livePhotoVideoId);
}
}
}
}

View File

@@ -90,8 +90,9 @@ export class StorageTemplateService {
}
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
if (asset.isReadOnly) {
this.logger.verbose(`Not moving read-only asset: ${asset.originalPath}`);
if (asset.isReadOnly || asset.isExternal) {
// External assets are not affected by storage template
// TODO: shouldn't this only apply to external assets?
return;
}

View File

@@ -1,4 +1,6 @@
import { Stats } from 'fs';
import { Readable } from 'stream';
import { CrawlOptionsDto } from '../library';
export interface ImmichReadStream {
stream: Readable;
@@ -30,4 +32,6 @@ export interface IStorageRepository {
mkdirSync(filepath: string): void;
checkDiskUsage(folder: string): Promise<DiskUsage>;
readdir(folder: string): Promise<string[]>;
stat(filepath: string): Promise<Stats>;
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
}

View File

@@ -70,4 +70,10 @@ export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.SIDECAR]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.LIBRARY]!: JobSettingsDto;
}

View File

@@ -51,6 +51,7 @@ export const defaults = Object.freeze<SystemConfig>({
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.LIBRARY]: { concurrency: 1 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },

View File

@@ -31,6 +31,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.LIBRARY]: { concurrency: 1 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },

View File

@@ -1,4 +1,4 @@
import { UserEntity } from '@app/infra/entities';
import { LibraryType, UserEntity } from '@app/infra/entities';
import {
BadRequestException,
ForbiddenException,
@@ -11,6 +11,7 @@ import fs from 'fs/promises';
import sanitize from 'sanitize-filename';
import { AuthUserDto } from '../auth';
import { ICryptoRepository } from '../crypto';
import { ILibraryRepository } from '../library/library.repository';
import { IUserRepository, UserListFilter } from './user.repository';
const SALT_ROUNDS = 10;
@@ -18,6 +19,7 @@ const SALT_ROUNDS = 10;
export class UserCore {
constructor(
private userRepository: IUserRepository,
private libraryRepository: ILibraryRepository,
private cryptoRepository: ICryptoRepository,
) {}
@@ -91,7 +93,19 @@ export class UserCore {
if (payload.storageLabel) {
payload.storageLabel = sanitize(payload.storageLabel);
}
return this.userRepository.create(payload);
const userEntity = await this.userRepository.create(payload);
await this.libraryRepository.create({
owner: { id: userEntity.id } as UserEntity,
name: 'Default Library',
assets: [],
type: LibraryType.UPLOAD,
importPaths: [],
exclusionPatterns: [],
isVisible: true,
});
return userEntity;
} catch (e) {
Logger.error(e, 'Create new user');
throw new InternalServerErrorException('Failed to register new user');

View File

@@ -10,6 +10,7 @@ import {
newAssetRepositoryMock,
newCryptoRepositoryMock,
newJobRepositoryMock,
newLibraryRepositoryMock,
newStorageRepositoryMock,
newUserRepositoryMock,
userStub,
@@ -20,6 +21,7 @@ import { IAssetRepository } from '../asset';
import { AuthUserDto } from '../auth';
import { ICryptoRepository } from '../crypto';
import { IJobRepository, JobName } from '../job';
import { ILibraryRepository } from '../library';
import { IStorageRepository } from '../storage';
import { UpdateUserDto } from './dto/update-user.dto';
import { UserResponseDto, mapUser } from './response-dto';
@@ -129,6 +131,7 @@ describe(UserService.name, () => {
let albumMock: jest.Mocked<IAlbumRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let libraryMock: jest.Mocked<ILibraryRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
beforeEach(async () => {
@@ -136,10 +139,11 @@ describe(UserService.name, () => {
albumMock = newAlbumRepositoryMock();
assetMock = newAssetRepositoryMock();
jobMock = newJobRepositoryMock();
libraryMock = newLibraryRepositoryMock();
storageMock = newStorageRepositoryMock();
userMock = newUserRepositoryMock();
sut = new UserService(userMock, cryptoRepositoryMock, albumMock, assetMock, jobMock, storageMock);
sut = new UserService(userMock, cryptoRepositoryMock, libraryMock, albumMock, assetMock, jobMock, storageMock);
when(userMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser);
when(userMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser);

View File

@@ -7,6 +7,7 @@ import { IAssetRepository } from '../asset/asset.repository';
import { AuthUserDto } from '../auth';
import { ICryptoRepository } from '../crypto/crypto.repository';
import { IEntityJob, IJobRepository, JobName } from '../job';
import { ILibraryRepository } from '../library/library.repository';
import { StorageCore, StorageFolder } from '../storage';
import { IStorageRepository } from '../storage/storage.repository';
import { CreateUserDto, UpdateUserDto, UserCountDto } from './dto';
@@ -30,13 +31,13 @@ export class UserService {
constructor(
@Inject(IUserRepository) private userRepository: IUserRepository,
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.userCore = new UserCore(userRepository, cryptoRepository);
this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
}
async getAll(authUser: AuthUserDto, isAll: boolean): Promise<UserResponseDto[]> {