mirror of
https://github.com/KevinMidboe/immich.git
synced 2025-12-08 12:19:05 +00:00
feat(server,web): libraries (#3124)
* feat: libraries Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
committed by
GitHub
parent
816db700e1
commit
acdc66413c
@@ -21,7 +21,13 @@ export enum Permission {
|
||||
|
||||
ARCHIVE_READ = 'archive.read',
|
||||
|
||||
TIMELINE_READ = 'timeline.read',
|
||||
TIMELINE_DOWNLOAD = 'timeline.download',
|
||||
|
||||
LIBRARY_CREATE = 'library.create',
|
||||
LIBRARY_READ = 'library.read',
|
||||
LIBRARY_UPDATE = 'library.update',
|
||||
LIBRARY_DELETE = 'library.delete',
|
||||
LIBRARY_DOWNLOAD = 'library.download',
|
||||
|
||||
PERSON_READ = 'person.read',
|
||||
@@ -165,12 +171,24 @@ export class AccessCore {
|
||||
case Permission.ARCHIVE_READ:
|
||||
return authUser.id === id;
|
||||
|
||||
case Permission.LIBRARY_READ:
|
||||
return authUser.id === id || (await this.repository.library.hasPartnerAccess(authUser.id, id));
|
||||
case Permission.TIMELINE_READ:
|
||||
return authUser.id === id || (await this.repository.timeline.hasPartnerAccess(authUser.id, id));
|
||||
|
||||
case Permission.LIBRARY_DOWNLOAD:
|
||||
case Permission.TIMELINE_DOWNLOAD:
|
||||
return authUser.id === id;
|
||||
|
||||
case Permission.LIBRARY_READ:
|
||||
return (
|
||||
(await this.repository.library.hasOwnerAccess(authUser.id, id)) ||
|
||||
(await this.repository.library.hasPartnerAccess(authUser.id, id))
|
||||
);
|
||||
|
||||
case Permission.LIBRARY_UPDATE:
|
||||
return this.repository.library.hasOwnerAccess(authUser.id, id);
|
||||
|
||||
case Permission.LIBRARY_DELETE:
|
||||
return this.repository.library.hasOwnerAccess(authUser.id, id);
|
||||
|
||||
case Permission.PERSON_READ:
|
||||
return this.repository.person.hasOwnerAccess(authUser.id, id);
|
||||
|
||||
|
||||
@@ -15,6 +15,11 @@ export interface IAccessRepository {
|
||||
};
|
||||
|
||||
library: {
|
||||
hasOwnerAccess(userId: string, libraryId: string): Promise<boolean>;
|
||||
hasPartnerAccess(userId: string, partnerId: string): Promise<boolean>;
|
||||
};
|
||||
|
||||
timeline: {
|
||||
hasPartnerAccess(userId: string, partnerId: string): Promise<boolean>;
|
||||
};
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ export enum WithoutProperty {
|
||||
|
||||
export enum WithProperty {
|
||||
SIDECAR = 'sidecar',
|
||||
IS_OFFLINE = 'isOffline',
|
||||
}
|
||||
|
||||
export enum TimeBucketSize {
|
||||
@@ -69,15 +70,18 @@ export interface TimeBucketItem {
|
||||
export const IAssetRepository = 'IAssetRepository';
|
||||
|
||||
export interface IAssetRepository {
|
||||
create(asset: Partial<AssetEntity>): Promise<AssetEntity>;
|
||||
getByDate(ownerId: string, date: Date): Promise<AssetEntity[]>;
|
||||
getByIds(ids: string[]): Promise<AssetEntity[]>;
|
||||
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
|
||||
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
|
||||
getByUserId(pagination: PaginationOptions, userId: string): Paginated<AssetEntity>;
|
||||
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
|
||||
getWith(pagination: PaginationOptions, property: WithProperty): Paginated<AssetEntity>;
|
||||
getWith(pagination: PaginationOptions, property: WithProperty, libraryId?: string): Paginated<AssetEntity>;
|
||||
getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
|
||||
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
|
||||
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]>;
|
||||
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
|
||||
deleteAll(ownerId: string): Promise<void>;
|
||||
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
|
||||
updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void>;
|
||||
@@ -87,5 +91,7 @@ export interface IAssetRepository {
|
||||
getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
|
||||
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
|
||||
getByTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
|
||||
remove(asset: AssetEntity): Promise<AssetEntity>;
|
||||
getById(assetId: string): Promise<AssetEntity>;
|
||||
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -501,6 +501,7 @@ describe(AssetService.name, () => {
|
||||
});
|
||||
|
||||
it('should return a list of archives (userId)', async () => {
|
||||
accessMock.library.hasOwnerAccess.mockResolvedValue(true);
|
||||
assetMock.getByUserId.mockResolvedValue({
|
||||
items: [assetStub.image, assetStub.video],
|
||||
hasNextPage: false,
|
||||
@@ -514,6 +515,8 @@ describe(AssetService.name, () => {
|
||||
});
|
||||
|
||||
it('should split archives by size', async () => {
|
||||
accessMock.library.hasOwnerAccess.mockResolvedValue(true);
|
||||
|
||||
assetMock.getByUserId.mockResolvedValue({
|
||||
items: [
|
||||
{ ...assetStub.image, id: 'asset-1' },
|
||||
|
||||
@@ -162,7 +162,7 @@ export class AssetService {
|
||||
if (dto.isArchived !== false) {
|
||||
await this.access.requirePermission(authUser, Permission.ARCHIVE_READ, [dto.userId]);
|
||||
}
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, [dto.userId]);
|
||||
await this.access.requirePermission(authUser, Permission.TIMELINE_READ, [dto.userId]);
|
||||
} else {
|
||||
dto.userId = authUser.id;
|
||||
}
|
||||
@@ -187,6 +187,10 @@ export class AssetService {
|
||||
throw new BadRequestException('Asset not found');
|
||||
}
|
||||
|
||||
if (asset.isOffline) {
|
||||
throw new BadRequestException('Asset is offline');
|
||||
}
|
||||
|
||||
return this.storageRepository.createReadStream(asset.originalPath, mimeTypes.lookup(asset.originalPath));
|
||||
}
|
||||
|
||||
@@ -268,7 +272,7 @@ export class AssetService {
|
||||
|
||||
if (dto.userId) {
|
||||
const userId = dto.userId;
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_DOWNLOAD, userId);
|
||||
await this.access.requirePermission(authUser, Permission.TIMELINE_DOWNLOAD, userId);
|
||||
return usePagination(PAGINATION_SIZE, (pagination) => this.assetRepository.getByUserId(pagination, userId));
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ export class AssetResponseDto {
|
||||
deviceId!: string;
|
||||
ownerId!: string;
|
||||
owner?: UserResponseDto;
|
||||
libraryId!: string;
|
||||
|
||||
@ApiProperty({ enumName: 'AssetTypeEnum', enum: AssetType })
|
||||
type!: AssetType;
|
||||
@@ -25,6 +26,9 @@ export class AssetResponseDto {
|
||||
updatedAt!: Date;
|
||||
isFavorite!: boolean;
|
||||
isArchived!: boolean;
|
||||
isOffline!: boolean;
|
||||
isExternal!: boolean;
|
||||
isReadOnly!: boolean;
|
||||
duration!: string;
|
||||
exifInfo?: ExifResponseDto;
|
||||
smartInfo?: SmartInfoResponseDto;
|
||||
@@ -42,6 +46,7 @@ function _map(entity: AssetEntity, withExif: boolean): AssetResponseDto {
|
||||
ownerId: entity.ownerId,
|
||||
owner: entity.owner ? mapUser(entity.owner) : undefined,
|
||||
deviceId: entity.deviceId,
|
||||
libraryId: entity.libraryId,
|
||||
type: entity.type,
|
||||
originalPath: entity.originalPath,
|
||||
originalFileName: entity.originalFileName,
|
||||
@@ -59,6 +64,9 @@ function _map(entity: AssetEntity, withExif: boolean): AssetResponseDto {
|
||||
tags: entity.tags?.map(mapTag),
|
||||
people: entity.faces?.map(mapFace).filter((person) => !person.isHidden),
|
||||
checksum: entity.checksum.toString('base64'),
|
||||
isExternal: entity.isExternal,
|
||||
isOffline: entity.isOffline,
|
||||
isReadOnly: entity.isReadOnly,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ export class AuditService {
|
||||
|
||||
async getDeletes(authUser: AuthUserDto, dto: AuditDeletesDto): Promise<AuditDeletesResponseDto> {
|
||||
const userId = dto.userId || authUser.id;
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
|
||||
await this.access.requirePermission(authUser, Permission.TIMELINE_READ, userId);
|
||||
|
||||
const audits = await this.repository.getAfter(dto.after, {
|
||||
ownerId: userId,
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
loginResponseStub,
|
||||
newCryptoRepositoryMock,
|
||||
newKeyRepositoryMock,
|
||||
newLibraryRepositoryMock,
|
||||
newSharedLinkRepositoryMock,
|
||||
newSystemConfigRepositoryMock,
|
||||
newUserRepositoryMock,
|
||||
@@ -20,6 +21,7 @@ import { Issuer, generators } from 'openid-client';
|
||||
import { Socket } from 'socket.io';
|
||||
import { IKeyRepository } from '../api-key';
|
||||
import { ICryptoRepository } from '../crypto/crypto.repository';
|
||||
import { ILibraryRepository } from '../library';
|
||||
import { ISharedLinkRepository } from '../shared-link';
|
||||
import { ISystemConfigRepository } from '../system-config';
|
||||
import { IUserRepository } from '../user';
|
||||
@@ -50,6 +52,7 @@ describe('AuthService', () => {
|
||||
let sut: AuthService;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
let userMock: jest.Mocked<IUserRepository>;
|
||||
let libraryMock: jest.Mocked<ILibraryRepository>;
|
||||
let configMock: jest.Mocked<ISystemConfigRepository>;
|
||||
let userTokenMock: jest.Mocked<IUserTokenRepository>;
|
||||
let shareMock: jest.Mocked<ISharedLinkRepository>;
|
||||
@@ -81,12 +84,13 @@ describe('AuthService', () => {
|
||||
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
userMock = newUserRepositoryMock();
|
||||
libraryMock = newLibraryRepositoryMock();
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
userTokenMock = newUserTokenRepositoryMock();
|
||||
shareMock = newSharedLinkRepositoryMock();
|
||||
keyMock = newKeyRepositoryMock();
|
||||
|
||||
sut = new AuthService(cryptoMock, configMock, userMock, userTokenMock, shareMock, keyMock);
|
||||
sut = new AuthService(cryptoMock, configMock, userMock, userTokenMock, libraryMock, shareMock, keyMock);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
|
||||
@@ -13,6 +13,7 @@ import { DateTime } from 'luxon';
|
||||
import { ClientMetadata, Issuer, UserinfoResponse, custom, generators } from 'openid-client';
|
||||
import { IKeyRepository } from '../api-key';
|
||||
import { ICryptoRepository } from '../crypto/crypto.repository';
|
||||
import { ILibraryRepository } from '../library';
|
||||
import { ISharedLinkRepository } from '../shared-link';
|
||||
import { ISystemConfigRepository } from '../system-config';
|
||||
import { SystemConfigCore } from '../system-config/system-config.core';
|
||||
@@ -66,11 +67,12 @@ export class AuthService {
|
||||
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
|
||||
@Inject(IUserRepository) userRepository: IUserRepository,
|
||||
@Inject(IUserTokenRepository) private userTokenRepository: IUserTokenRepository,
|
||||
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
|
||||
@Inject(ISharedLinkRepository) private sharedLinkRepository: ISharedLinkRepository,
|
||||
@Inject(IKeyRepository) private keyRepository: IKeyRepository,
|
||||
) {
|
||||
this.configCore = new SystemConfigCore(configRepository);
|
||||
this.userCore = new UserCore(userRepository, cryptoRepository);
|
||||
this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
|
||||
|
||||
custom.setHttpOptionsDefaults({ timeout: 30000 });
|
||||
}
|
||||
|
||||
@@ -102,6 +102,7 @@ export const mimeTypes = {
|
||||
video,
|
||||
|
||||
isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
|
||||
isImage: (filename: string) => isType(filename, image),
|
||||
isProfile: (filename: string) => isType(filename, profile),
|
||||
isSidecar: (filename: string) => isType(filename, sidecar),
|
||||
isVideo: (filename: string) => isType(filename, video),
|
||||
@@ -115,4 +116,5 @@ export const mimeTypes = {
|
||||
}
|
||||
return AssetType.OTHER;
|
||||
},
|
||||
getSupportedFileExtensions: () => Object.keys(image).concat(Object.keys(video)),
|
||||
};
|
||||
|
||||
@@ -6,6 +6,7 @@ import { AuditService } from './audit';
|
||||
import { AuthService } from './auth';
|
||||
import { FacialRecognitionService } from './facial-recognition';
|
||||
import { JobService } from './job';
|
||||
import { LibraryService } from './library';
|
||||
import { MediaService } from './media';
|
||||
import { MetadataService } from './metadata';
|
||||
import { PartnerService } from './partner';
|
||||
@@ -30,6 +31,7 @@ const providers: Provider[] = [
|
||||
JobService,
|
||||
MediaService,
|
||||
MetadataService,
|
||||
LibraryService,
|
||||
PersonService,
|
||||
PartnerService,
|
||||
SearchService,
|
||||
|
||||
@@ -12,6 +12,7 @@ export * from './domain.module';
|
||||
export * from './domain.util';
|
||||
export * from './facial-recognition';
|
||||
export * from './job';
|
||||
export * from './library';
|
||||
export * from './media';
|
||||
export * from './metadata';
|
||||
export * from './partner';
|
||||
|
||||
@@ -9,6 +9,7 @@ export enum QueueName {
|
||||
STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
|
||||
SEARCH = 'search',
|
||||
SIDECAR = 'sidecar',
|
||||
LIBRARY = 'library',
|
||||
}
|
||||
|
||||
export enum JobCommand {
|
||||
@@ -53,6 +54,15 @@ export enum JobName {
|
||||
RECOGNIZE_FACES = 'recognize-faces',
|
||||
PERSON_CLEANUP = 'person-cleanup',
|
||||
|
||||
// library managment
|
||||
LIBRARY_SCAN = 'library-refresh',
|
||||
LIBRARY_SCAN_ASSET = 'library-refresh-asset',
|
||||
LIBRARY_REMOVE_OFFLINE = 'library-remove-offline',
|
||||
LIBRARY_MARK_ASSET_OFFLINE = 'library-mark-asset-offline',
|
||||
LIBRARY_DELETE = 'library-delete',
|
||||
LIBRARY_QUEUE_SCAN_ALL = 'library-queue-all-refresh',
|
||||
LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup',
|
||||
|
||||
// cleanup
|
||||
DELETE_FILES = 'delete-files',
|
||||
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
|
||||
@@ -140,4 +150,13 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
|
||||
[JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
|
||||
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,
|
||||
[JobName.SIDECAR_SYNC]: QueueName.SIDECAR,
|
||||
|
||||
// Library managment
|
||||
[JobName.LIBRARY_SCAN_ASSET]: QueueName.LIBRARY,
|
||||
[JobName.LIBRARY_MARK_ASSET_OFFLINE]: QueueName.LIBRARY,
|
||||
[JobName.LIBRARY_SCAN]: QueueName.LIBRARY,
|
||||
[JobName.LIBRARY_DELETE]: QueueName.LIBRARY,
|
||||
[JobName.LIBRARY_REMOVE_OFFLINE]: QueueName.LIBRARY,
|
||||
[JobName.LIBRARY_QUEUE_SCAN_ALL]: QueueName.LIBRARY,
|
||||
[JobName.LIBRARY_QUEUE_CLEANUP]: QueueName.LIBRARY,
|
||||
};
|
||||
|
||||
@@ -79,4 +79,7 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
|
||||
|
||||
@ApiProperty({ type: JobStatusDto })
|
||||
[QueueName.SIDECAR]!: JobStatusDto;
|
||||
|
||||
@ApiProperty({ type: JobStatusDto })
|
||||
[QueueName.LIBRARY]!: JobStatusDto;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,21 @@ export interface IEntityJob extends IBaseJob {
|
||||
source?: 'upload';
|
||||
}
|
||||
|
||||
export interface IOfflineLibraryFileJob extends IEntityJob {
|
||||
assetPath: string;
|
||||
}
|
||||
|
||||
export interface ILibraryFileJob extends IEntityJob {
|
||||
ownerId: string;
|
||||
assetPath: string;
|
||||
forceRefresh: boolean;
|
||||
}
|
||||
|
||||
export interface ILibraryRefreshJob extends IEntityJob {
|
||||
refreshModifiedFiles: boolean;
|
||||
refreshAllFiles: boolean;
|
||||
}
|
||||
|
||||
export interface IBulkEntityJob extends IBaseJob {
|
||||
ids: string[];
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { JobName, QueueName } from './job.constants';
|
||||
|
||||
import {
|
||||
IAssetFaceJob,
|
||||
IBaseJob,
|
||||
@@ -6,6 +7,9 @@ import {
|
||||
IDeleteFilesJob,
|
||||
IEntityJob,
|
||||
IFaceThumbnailJob,
|
||||
ILibraryFileJob,
|
||||
ILibraryRefreshJob,
|
||||
IOfflineLibraryFileJob,
|
||||
} from './job.interface';
|
||||
|
||||
export interface JobCounts {
|
||||
@@ -74,6 +78,15 @@ export type JobItem =
|
||||
// Asset Deletion
|
||||
| { name: JobName.PERSON_CLEANUP; data?: IBaseJob }
|
||||
|
||||
// Library Managment
|
||||
| { name: JobName.LIBRARY_SCAN_ASSET; data: ILibraryFileJob }
|
||||
| { name: JobName.LIBRARY_MARK_ASSET_OFFLINE; data: IOfflineLibraryFileJob }
|
||||
| { name: JobName.LIBRARY_SCAN; data: ILibraryRefreshJob }
|
||||
| { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
|
||||
| { name: JobName.LIBRARY_DELETE; data: IEntityJob }
|
||||
| { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
|
||||
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }
|
||||
|
||||
// Search
|
||||
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
|
||||
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }
|
||||
|
||||
@@ -52,6 +52,7 @@ describe(JobService.name, () => {
|
||||
[{ name: JobName.PERSON_CLEANUP }],
|
||||
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
|
||||
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
|
||||
[{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }],
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -97,6 +98,7 @@ describe(JobService.name, () => {
|
||||
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
|
||||
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
|
||||
[QueueName.SIDECAR]: expectedJobStatus,
|
||||
[QueueName.LIBRARY]: expectedJobStatus,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -225,6 +227,7 @@ describe(JobService.name, () => {
|
||||
[QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
|
||||
[QueueName.SEARCH]: { concurrency: 10 },
|
||||
[QueueName.SIDECAR]: { concurrency: 10 },
|
||||
[QueueName.LIBRARY]: { concurrency: 10 },
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
|
||||
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
|
||||
[QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
|
||||
@@ -237,6 +240,7 @@ describe(JobService.name, () => {
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.OBJECT_TAGGING, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
|
||||
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
|
||||
|
||||
@@ -98,6 +98,9 @@ export class JobService {
|
||||
await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
|
||||
return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } });
|
||||
|
||||
case QueueName.LIBRARY:
|
||||
return this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force } });
|
||||
|
||||
default:
|
||||
throw new BadRequestException(`Invalid job name: ${name}`);
|
||||
}
|
||||
@@ -118,7 +121,7 @@ export class JobService {
|
||||
await this.onDone(item);
|
||||
}
|
||||
} catch (error: Error | any) {
|
||||
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, data);
|
||||
this.logger.error(`Unable to run job handler (${queueName}/${name}): ${error}`, error?.stack, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -138,6 +141,7 @@ export class JobService {
|
||||
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
|
||||
await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
|
||||
await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
|
||||
await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } });
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
3
server/src/domain/library/index.ts
Normal file
3
server/src/domain/library/index.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export * from './library.dto';
|
||||
export * from './library.repository';
|
||||
export * from './library.service';
|
||||
124
server/src/domain/library/library.dto.ts
Normal file
124
server/src/domain/library/library.dto.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
import { LibraryEntity, LibraryType } from '@app/infra/entities';
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator';
|
||||
import { ValidateUUID } from '../domain.util';
|
||||
|
||||
export class CreateLibraryDto {
|
||||
@IsEnum(LibraryType)
|
||||
@ApiProperty({ enumName: 'LibraryType', enum: LibraryType })
|
||||
type!: LibraryType;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
@IsNotEmpty()
|
||||
name?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
isVisible?: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsString({ each: true })
|
||||
@IsNotEmpty({ each: true })
|
||||
importPaths?: string[];
|
||||
|
||||
@IsOptional()
|
||||
@IsString({ each: true })
|
||||
@IsNotEmpty({ each: true })
|
||||
exclusionPatterns?: string[];
|
||||
}
|
||||
|
||||
export class UpdateLibraryDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
name?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
isVisible?: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsString({ each: true })
|
||||
@IsNotEmpty({ each: true })
|
||||
importPaths?: string[];
|
||||
|
||||
@IsOptional()
|
||||
@IsNotEmpty({ each: true })
|
||||
@IsString({ each: true })
|
||||
exclusionPatterns?: string[];
|
||||
}
|
||||
|
||||
export class CrawlOptionsDto {
|
||||
pathsToCrawl!: string[];
|
||||
includeHidden? = false;
|
||||
exclusionPatterns?: string[];
|
||||
}
|
||||
|
||||
export class LibrarySearchDto {
|
||||
@ValidateUUID({ optional: true })
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
export class ScanLibraryDto {
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
refreshModifiedFiles?: boolean;
|
||||
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
refreshAllFiles?: boolean = false;
|
||||
}
|
||||
|
||||
export class LibraryResponseDto {
|
||||
id!: string;
|
||||
ownerId!: string;
|
||||
name!: string;
|
||||
|
||||
@ApiProperty({ enumName: 'LibraryType', enum: LibraryType })
|
||||
type!: LibraryType;
|
||||
|
||||
@ApiProperty({ type: 'integer' })
|
||||
assetCount!: number;
|
||||
|
||||
importPaths!: string[];
|
||||
|
||||
exclusionPatterns!: string[];
|
||||
|
||||
createdAt!: Date;
|
||||
updatedAt!: Date;
|
||||
refreshedAt!: Date | null;
|
||||
}
|
||||
|
||||
export class LibraryStatsResponseDto {
|
||||
@ApiProperty({ type: 'integer' })
|
||||
photos = 0;
|
||||
|
||||
@ApiProperty({ type: 'integer' })
|
||||
videos = 0;
|
||||
|
||||
@ApiProperty({ type: 'integer' })
|
||||
total = 0;
|
||||
|
||||
@ApiProperty({ type: 'integer', format: 'int64' })
|
||||
usage = 0;
|
||||
}
|
||||
|
||||
export function mapLibrary(entity: LibraryEntity): LibraryResponseDto {
|
||||
let assetCount = 0;
|
||||
if (entity.assets) {
|
||||
assetCount = entity.assets.length;
|
||||
}
|
||||
return {
|
||||
id: entity.id,
|
||||
ownerId: entity.ownerId,
|
||||
type: entity.type,
|
||||
name: entity.name,
|
||||
createdAt: entity.createdAt,
|
||||
updatedAt: entity.updatedAt,
|
||||
refreshedAt: entity.refreshedAt,
|
||||
assetCount,
|
||||
importPaths: entity.importPaths,
|
||||
exclusionPatterns: entity.exclusionPatterns,
|
||||
};
|
||||
}
|
||||
22
server/src/domain/library/library.repository.ts
Normal file
22
server/src/domain/library/library.repository.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { LibraryEntity, LibraryType } from '@app/infra/entities';
|
||||
import { LibraryStatsResponseDto } from './library.dto';
|
||||
|
||||
export const ILibraryRepository = 'ILibraryRepository';
|
||||
|
||||
export interface ILibraryRepository {
|
||||
getCountForUser(ownerId: string): Promise<number>;
|
||||
getAllByUserId(userId: string, type?: LibraryType): Promise<LibraryEntity[]>;
|
||||
getAll(withDeleted?: boolean, type?: LibraryType): Promise<LibraryEntity[]>;
|
||||
getAllDeleted(): Promise<LibraryEntity[]>;
|
||||
get(id: string, withDeleted?: boolean): Promise<LibraryEntity | null>;
|
||||
create(library: Partial<LibraryEntity>): Promise<LibraryEntity>;
|
||||
delete(id: string): Promise<void>;
|
||||
softDelete(id: string): Promise<void>;
|
||||
getDefaultUploadLibrary(ownerId: string): Promise<LibraryEntity | null>;
|
||||
getUploadLibraryCount(ownerId: string): Promise<number>;
|
||||
update(library: Partial<LibraryEntity>): Promise<LibraryEntity>;
|
||||
getStatistics(id: string): Promise<LibraryStatsResponseDto>;
|
||||
getOnlineAssetPaths(id: string): Promise<string[]>;
|
||||
getAssetIds(id: string, withDeleted?: boolean): Promise<string[]>;
|
||||
existsByName(name: string, withDeleted?: boolean): Promise<boolean>;
|
||||
}
|
||||
1204
server/src/domain/library/library.service.spec.ts
Normal file
1204
server/src/domain/library/library.service.spec.ts
Normal file
File diff suppressed because it is too large
Load Diff
468
server/src/domain/library/library.service.ts
Normal file
468
server/src/domain/library/library.service.ts
Normal file
@@ -0,0 +1,468 @@
|
||||
import { AssetType, LibraryType } from '@app/infra/entities';
|
||||
import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { R_OK } from 'node:constants';
|
||||
import { Stats } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { basename, parse } from 'path';
|
||||
import { AccessCore, IAccessRepository, Permission } from '../access';
|
||||
import { IAssetRepository, WithProperty } from '../asset';
|
||||
import { AuthUserDto } from '../auth';
|
||||
import { usePagination } from '../domain.util';
|
||||
|
||||
import { ICryptoRepository } from '../crypto';
|
||||
import { mimeTypes } from '../domain.constant';
|
||||
import {
|
||||
IBaseJob,
|
||||
IEntityJob,
|
||||
IJobRepository,
|
||||
ILibraryFileJob,
|
||||
ILibraryRefreshJob,
|
||||
IOfflineLibraryFileJob,
|
||||
JobName,
|
||||
JOBS_ASSET_PAGINATION_SIZE,
|
||||
} from '../job';
|
||||
import { IStorageRepository } from '../storage';
|
||||
import { IUserRepository } from '../user';
|
||||
import {
|
||||
CreateLibraryDto,
|
||||
LibraryResponseDto,
|
||||
LibraryStatsResponseDto,
|
||||
mapLibrary,
|
||||
ScanLibraryDto,
|
||||
UpdateLibraryDto,
|
||||
} from './library.dto';
|
||||
import { ILibraryRepository } from './library.repository';
|
||||
|
||||
@Injectable()
|
||||
export class LibraryService {
|
||||
readonly logger = new Logger(LibraryService.name);
|
||||
private access: AccessCore;
|
||||
|
||||
constructor(
|
||||
@Inject(IAccessRepository) accessRepository: IAccessRepository,
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
@Inject(ILibraryRepository) private repository: ILibraryRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
@Inject(IUserRepository) private userRepository: IUserRepository,
|
||||
) {
|
||||
this.access = new AccessCore(accessRepository);
|
||||
}
|
||||
|
||||
async getStatistics(authUser: AuthUserDto, id: string): Promise<LibraryStatsResponseDto> {
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, id);
|
||||
return this.repository.getStatistics(id);
|
||||
}
|
||||
|
||||
async getCount(authUser: AuthUserDto): Promise<number> {
|
||||
return this.repository.getCountForUser(authUser.id);
|
||||
}
|
||||
|
||||
async getAllForUser(authUser: AuthUserDto): Promise<LibraryResponseDto[]> {
|
||||
const libraries = await this.repository.getAllByUserId(authUser.id);
|
||||
return libraries.map((library) => mapLibrary(library));
|
||||
}
|
||||
|
||||
async get(authUser: AuthUserDto, id: string): Promise<LibraryResponseDto> {
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, id);
|
||||
const library = await this.findOrFail(id);
|
||||
return mapLibrary(library);
|
||||
}
|
||||
|
||||
async handleQueueCleanup(): Promise<boolean> {
|
||||
this.logger.debug('Cleaning up any pending library deletions');
|
||||
const pendingDeletion = await this.repository.getAllDeleted();
|
||||
for (const libraryToDelete of pendingDeletion) {
|
||||
await this.jobRepository.queue({ name: JobName.LIBRARY_DELETE, data: { id: libraryToDelete.id } });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async create(authUser: AuthUserDto, dto: CreateLibraryDto): Promise<LibraryResponseDto> {
|
||||
switch (dto.type) {
|
||||
case LibraryType.EXTERNAL:
|
||||
if (!dto.name) {
|
||||
dto.name = 'New External Library';
|
||||
}
|
||||
break;
|
||||
case LibraryType.UPLOAD:
|
||||
if (!dto.name) {
|
||||
dto.name = 'New Upload Library';
|
||||
}
|
||||
if (dto.importPaths && dto.importPaths.length > 0) {
|
||||
throw new BadRequestException('Upload libraries cannot have import paths');
|
||||
}
|
||||
if (dto.exclusionPatterns && dto.exclusionPatterns.length > 0) {
|
||||
throw new BadRequestException('Upload libraries cannot have exclusion patterns');
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
const library = await this.repository.create({
|
||||
ownerId: authUser.id,
|
||||
name: dto.name,
|
||||
type: dto.type,
|
||||
importPaths: dto.importPaths ?? [],
|
||||
exclusionPatterns: dto.exclusionPatterns ?? [],
|
||||
isVisible: dto.isVisible ?? true,
|
||||
});
|
||||
|
||||
return mapLibrary(library);
|
||||
}
|
||||
|
||||
async update(authUser: AuthUserDto, id: string, dto: UpdateLibraryDto): Promise<LibraryResponseDto> {
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_UPDATE, id);
|
||||
const library = await this.repository.update({ id, ...dto });
|
||||
return mapLibrary(library);
|
||||
}
|
||||
|
||||
async delete(authUser: AuthUserDto, id: string) {
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_DELETE, id);
|
||||
|
||||
const library = await this.findOrFail(id);
|
||||
const uploadCount = await this.repository.getUploadLibraryCount(authUser.id);
|
||||
if (library.type === LibraryType.UPLOAD && uploadCount <= 1) {
|
||||
throw new BadRequestException('Cannot delete the last upload library');
|
||||
}
|
||||
|
||||
await this.repository.softDelete(id);
|
||||
await this.jobRepository.queue({ name: JobName.LIBRARY_DELETE, data: { id } });
|
||||
}
|
||||
|
||||
async handleDeleteLibrary(job: IEntityJob): Promise<boolean> {
|
||||
const library = await this.repository.get(job.id, true);
|
||||
if (!library) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO use pagination
|
||||
const assetIds = await this.repository.getAssetIds(job.id);
|
||||
this.logger.debug(`Will delete ${assetIds.length} asset(s) in library ${job.id}`);
|
||||
// TODO queue a job for asset deletion
|
||||
await this.deleteAssets(assetIds);
|
||||
this.logger.log(`Deleting library ${job.id}`);
|
||||
await this.repository.delete(job.id);
|
||||
return true;
|
||||
}
|
||||
|
||||
async handleAssetRefresh(job: ILibraryFileJob) {
|
||||
const assetPath = path.normalize(job.assetPath);
|
||||
|
||||
const user = await this.userRepository.get(job.ownerId);
|
||||
if (!user?.externalPath) {
|
||||
this.logger.warn('User has no external path set, cannot import asset');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!path.normalize(assetPath).match(new RegExp(`^${user.externalPath}`))) {
|
||||
this.logger.error("Asset must be within the user's external path");
|
||||
return false;
|
||||
}
|
||||
|
||||
const existingAssetEntity = await this.assetRepository.getByLibraryIdAndOriginalPath(job.id, assetPath);
|
||||
|
||||
let stats: Stats;
|
||||
try {
|
||||
stats = await this.storageRepository.stat(assetPath);
|
||||
} catch (error: Error | any) {
|
||||
// Can't access file, probably offline
|
||||
if (existingAssetEntity) {
|
||||
// Mark asset as offline
|
||||
this.logger.debug(`Marking asset as offline: ${assetPath}`);
|
||||
|
||||
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: true });
|
||||
return true;
|
||||
} else {
|
||||
// File can't be accessed and does not already exist in db
|
||||
throw new BadRequestException("Can't access file", { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
let doImport = false;
|
||||
let doRefresh = false;
|
||||
|
||||
if (job.forceRefresh) {
|
||||
doRefresh = true;
|
||||
}
|
||||
|
||||
if (!existingAssetEntity) {
|
||||
// This asset is new to us, read it from disk
|
||||
this.logger.debug(`Importing new asset: ${assetPath}`);
|
||||
doImport = true;
|
||||
} else if (stats.mtime.toISOString() !== existingAssetEntity.fileModifiedAt.toISOString()) {
|
||||
// File modification time has changed since last time we checked, re-read from disk
|
||||
this.logger.debug(
|
||||
`File modification time has changed, re-importing asset: ${assetPath}. Old mtime: ${existingAssetEntity.fileModifiedAt}. New mtime: ${stats.mtime}`,
|
||||
);
|
||||
doRefresh = true;
|
||||
} else if (!job.forceRefresh && stats && !existingAssetEntity.isOffline) {
|
||||
// Asset exists on disk and in db and mtime has not changed. Also, we are not forcing refresn. Therefore, do nothing
|
||||
this.logger.debug(`Asset already exists in database and on disk, will not import: ${assetPath}`);
|
||||
}
|
||||
|
||||
if (stats && existingAssetEntity?.isOffline) {
|
||||
// File was previously offline but is now online
|
||||
this.logger.debug(`Marking previously-offline asset as online: ${assetPath}`);
|
||||
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: false });
|
||||
doRefresh = true;
|
||||
}
|
||||
|
||||
if (!doImport && !doRefresh) {
|
||||
// If we don't import, exit here
|
||||
return true;
|
||||
}
|
||||
|
||||
let assetType: AssetType;
|
||||
|
||||
if (mimeTypes.isImage(assetPath)) {
|
||||
assetType = AssetType.IMAGE;
|
||||
} else if (mimeTypes.isVideo(assetPath)) {
|
||||
assetType = AssetType.VIDEO;
|
||||
} else {
|
||||
throw new BadRequestException(`Unsupported file type ${assetPath}`);
|
||||
}
|
||||
|
||||
// TODO: doesn't xmp replace the file extension? Will need investigation
|
||||
let sidecarPath: string | null = null;
|
||||
if (await this.storageRepository.checkFileExists(`${assetPath}.xmp`, R_OK)) {
|
||||
sidecarPath = `${assetPath}.xmp`;
|
||||
}
|
||||
|
||||
const deviceAssetId = `${basename(assetPath)}`.replace(/\s+/g, '');
|
||||
|
||||
const pathHash = this.cryptoRepository.hashSha1(`path:${assetPath}`);
|
||||
|
||||
let assetId;
|
||||
if (doImport) {
|
||||
const library = await this.repository.get(job.id, true);
|
||||
if (library?.deletedAt) {
|
||||
this.logger.error('Cannot import asset into deleted library');
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO: In wait of refactoring the domain asset service, this function is just manually written like this
|
||||
const addedAsset = await this.assetRepository.create({
|
||||
ownerId: job.ownerId,
|
||||
libraryId: job.id,
|
||||
checksum: pathHash,
|
||||
originalPath: assetPath,
|
||||
deviceAssetId: deviceAssetId,
|
||||
deviceId: 'Library Import',
|
||||
fileCreatedAt: stats.ctime,
|
||||
fileModifiedAt: stats.mtime,
|
||||
type: assetType,
|
||||
originalFileName: parse(assetPath).name,
|
||||
sidecarPath,
|
||||
isReadOnly: true,
|
||||
isExternal: true,
|
||||
});
|
||||
assetId = addedAsset.id;
|
||||
} else if (doRefresh && existingAssetEntity) {
|
||||
assetId = existingAssetEntity.id;
|
||||
await this.assetRepository.updateAll([existingAssetEntity.id], {
|
||||
fileCreatedAt: stats.ctime,
|
||||
fileModifiedAt: stats.mtime,
|
||||
});
|
||||
} else {
|
||||
// Not importing and not refreshing, do nothing
|
||||
return true;
|
||||
}
|
||||
|
||||
this.logger.debug(`Queuing metadata extraction for: ${assetPath}`);
|
||||
|
||||
await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: assetId, source: 'upload' } });
|
||||
|
||||
if (assetType === AssetType.VIDEO) {
|
||||
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id: assetId } });
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async queueScan(authUser: AuthUserDto, id: string, dto: ScanLibraryDto) {
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_UPDATE, id);
|
||||
|
||||
const library = await this.repository.get(id);
|
||||
if (!library || library.type !== LibraryType.EXTERNAL) {
|
||||
throw new BadRequestException('Can only refresh external libraries');
|
||||
}
|
||||
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.LIBRARY_SCAN,
|
||||
data: {
|
||||
id,
|
||||
refreshModifiedFiles: dto.refreshModifiedFiles ?? false,
|
||||
refreshAllFiles: dto.refreshAllFiles ?? false,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async queueRemoveOffline(authUser: AuthUserDto, id: string) {
|
||||
this.logger.verbose(`Removing offline files from library: ${id}`);
|
||||
await this.access.requirePermission(authUser, Permission.LIBRARY_UPDATE, id);
|
||||
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.LIBRARY_REMOVE_OFFLINE,
|
||||
data: {
|
||||
id,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async handleQueueAllScan(job: IBaseJob): Promise<boolean> {
|
||||
this.logger.debug(`Refreshing all external libraries: force=${job.force}`);
|
||||
|
||||
// Queue cleanup
|
||||
await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_CLEANUP, data: {} });
|
||||
|
||||
// Queue all library refresh
|
||||
const libraries = await this.repository.getAll(true, LibraryType.EXTERNAL);
|
||||
for (const library of libraries) {
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.LIBRARY_SCAN,
|
||||
data: {
|
||||
id: library.id,
|
||||
refreshModifiedFiles: !job.force,
|
||||
refreshAllFiles: job.force ?? false,
|
||||
},
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async handleOfflineRemoval(job: IEntityJob): Promise<boolean> {
|
||||
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
|
||||
return this.assetRepository.getWith(pagination, WithProperty.IS_OFFLINE, job.id);
|
||||
});
|
||||
|
||||
const assetIds: string[] = [];
|
||||
|
||||
for await (const assets of assetPagination) {
|
||||
for (const asset of assets) {
|
||||
assetIds.push(asset.id);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.verbose(`Found ${assetIds.length} offline assets to remove`);
|
||||
await this.deleteAssets(assetIds);
|
||||
return true;
|
||||
}
|
||||
|
||||
async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise<boolean> {
|
||||
const library = await this.repository.get(job.id);
|
||||
if (!library || library.type !== LibraryType.EXTERNAL) {
|
||||
this.logger.warn('Can only refresh external libraries');
|
||||
return false;
|
||||
}
|
||||
|
||||
const user = await this.userRepository.get(library.ownerId);
|
||||
if (!user?.externalPath) {
|
||||
this.logger.warn('User has no external path set, cannot refresh library');
|
||||
return false;
|
||||
}
|
||||
|
||||
this.logger.verbose(`Refreshing library: ${job.id}`);
|
||||
const crawledAssetPaths = (
|
||||
await this.storageRepository.crawl({
|
||||
pathsToCrawl: library.importPaths,
|
||||
exclusionPatterns: library.exclusionPatterns,
|
||||
})
|
||||
)
|
||||
.map(path.normalize)
|
||||
.filter((assetPath) =>
|
||||
// Filter out paths that are not within the user's external path
|
||||
assetPath.match(new RegExp(`^${user.externalPath}`)),
|
||||
);
|
||||
|
||||
this.logger.debug(`Found ${crawledAssetPaths.length} assets when crawling import paths ${library.importPaths}`);
|
||||
const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]);
|
||||
const offlineAssets = assetsInLibrary.filter((asset) => !crawledAssetPaths.includes(asset.originalPath));
|
||||
this.logger.debug(`${offlineAssets.length} assets in library are not present on disk and will be marked offline`);
|
||||
|
||||
for (const offlineAsset of offlineAssets) {
|
||||
const offlineJobData: IOfflineLibraryFileJob = {
|
||||
id: job.id,
|
||||
assetPath: offlineAsset.originalPath,
|
||||
};
|
||||
|
||||
await this.jobRepository.queue({ name: JobName.LIBRARY_MARK_ASSET_OFFLINE, data: offlineJobData });
|
||||
}
|
||||
|
||||
if (crawledAssetPaths.length > 0) {
|
||||
let filteredPaths: string[] = [];
|
||||
if (job.refreshAllFiles || job.refreshModifiedFiles) {
|
||||
filteredPaths = crawledAssetPaths;
|
||||
} else {
|
||||
const existingPaths = await this.repository.getOnlineAssetPaths(job.id);
|
||||
this.logger.debug(`Found ${existingPaths.length} existing asset(s) in library ${job.id}`);
|
||||
|
||||
filteredPaths = crawledAssetPaths.filter((assetPath) => !existingPaths.includes(assetPath));
|
||||
this.logger.debug(`After db comparison, ${filteredPaths.length} asset(s) remain to be imported`);
|
||||
}
|
||||
|
||||
for (const assetPath of filteredPaths) {
|
||||
const libraryJobData: ILibraryFileJob = {
|
||||
id: job.id,
|
||||
assetPath: path.normalize(assetPath),
|
||||
ownerId: library.ownerId,
|
||||
forceRefresh: job.refreshAllFiles ?? false,
|
||||
};
|
||||
|
||||
await this.jobRepository.queue({ name: JobName.LIBRARY_SCAN_ASSET, data: libraryJobData });
|
||||
}
|
||||
}
|
||||
|
||||
await this.repository.update({ id: job.id, refreshedAt: new Date() });
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async handleOfflineAsset(job: IOfflineLibraryFileJob): Promise<boolean> {
|
||||
const existingAssetEntity = await this.assetRepository.getByLibraryIdAndOriginalPath(job.id, job.assetPath);
|
||||
|
||||
if (existingAssetEntity) {
|
||||
this.logger.verbose(`Marking asset as offline: ${job.assetPath}`);
|
||||
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: true });
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async findOrFail(id: string) {
|
||||
const library = await this.repository.get(id);
|
||||
if (!library) {
|
||||
throw new BadRequestException('Library not found');
|
||||
}
|
||||
return library;
|
||||
}
|
||||
|
||||
private async deleteAssets(assetIds: string[]) {
|
||||
// TODO: this should be refactored to a centralized asset deletion service
|
||||
for (const assetId of assetIds) {
|
||||
const asset = await this.assetRepository.getById(assetId);
|
||||
this.logger.debug(`Removing asset from library: ${asset.originalPath}`);
|
||||
|
||||
if (asset.faces) {
|
||||
await Promise.all(
|
||||
asset.faces.map(({ assetId, personId }) =>
|
||||
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
await this.assetRepository.remove(asset);
|
||||
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [asset.id] } });
|
||||
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.DELETE_FILES,
|
||||
data: { files: [asset.webpPath, asset.resizePath, asset.encodedVideoPath, asset.sidecarPath] },
|
||||
});
|
||||
|
||||
// TODO refactor this to use cascades
|
||||
if (asset.livePhotoVideoId && !assetIds.includes(asset.livePhotoVideoId)) {
|
||||
assetIds.push(asset.livePhotoVideoId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -90,8 +90,9 @@ export class StorageTemplateService {
|
||||
}
|
||||
|
||||
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
|
||||
if (asset.isReadOnly) {
|
||||
this.logger.verbose(`Not moving read-only asset: ${asset.originalPath}`);
|
||||
if (asset.isReadOnly || asset.isExternal) {
|
||||
// External assets are not affected by storage template
|
||||
// TODO: shouldn't this only apply to external assets?
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Stats } from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import { CrawlOptionsDto } from '../library';
|
||||
|
||||
export interface ImmichReadStream {
|
||||
stream: Readable;
|
||||
@@ -30,4 +32,6 @@ export interface IStorageRepository {
|
||||
mkdirSync(filepath: string): void;
|
||||
checkDiskUsage(folder: string): Promise<DiskUsage>;
|
||||
readdir(folder: string): Promise<string[]>;
|
||||
stat(filepath: string): Promise<Stats>;
|
||||
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
|
||||
}
|
||||
|
||||
@@ -70,4 +70,10 @@ export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.SIDECAR]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.LIBRARY]!: JobSettingsDto;
|
||||
}
|
||||
|
||||
@@ -51,6 +51,7 @@ export const defaults = Object.freeze<SystemConfig>({
|
||||
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
|
||||
[QueueName.SEARCH]: { concurrency: 5 },
|
||||
[QueueName.SIDECAR]: { concurrency: 5 },
|
||||
[QueueName.LIBRARY]: { concurrency: 1 },
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
|
||||
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
|
||||
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
|
||||
|
||||
@@ -31,6 +31,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
||||
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
|
||||
[QueueName.SEARCH]: { concurrency: 5 },
|
||||
[QueueName.SIDECAR]: { concurrency: 5 },
|
||||
[QueueName.LIBRARY]: { concurrency: 1 },
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
|
||||
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
|
||||
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { UserEntity } from '@app/infra/entities';
|
||||
import { LibraryType, UserEntity } from '@app/infra/entities';
|
||||
import {
|
||||
BadRequestException,
|
||||
ForbiddenException,
|
||||
@@ -11,6 +11,7 @@ import fs from 'fs/promises';
|
||||
import sanitize from 'sanitize-filename';
|
||||
import { AuthUserDto } from '../auth';
|
||||
import { ICryptoRepository } from '../crypto';
|
||||
import { ILibraryRepository } from '../library/library.repository';
|
||||
import { IUserRepository, UserListFilter } from './user.repository';
|
||||
|
||||
const SALT_ROUNDS = 10;
|
||||
@@ -18,6 +19,7 @@ const SALT_ROUNDS = 10;
|
||||
export class UserCore {
|
||||
constructor(
|
||||
private userRepository: IUserRepository,
|
||||
private libraryRepository: ILibraryRepository,
|
||||
private cryptoRepository: ICryptoRepository,
|
||||
) {}
|
||||
|
||||
@@ -91,7 +93,19 @@ export class UserCore {
|
||||
if (payload.storageLabel) {
|
||||
payload.storageLabel = sanitize(payload.storageLabel);
|
||||
}
|
||||
return this.userRepository.create(payload);
|
||||
|
||||
const userEntity = await this.userRepository.create(payload);
|
||||
await this.libraryRepository.create({
|
||||
owner: { id: userEntity.id } as UserEntity,
|
||||
name: 'Default Library',
|
||||
assets: [],
|
||||
type: LibraryType.UPLOAD,
|
||||
importPaths: [],
|
||||
exclusionPatterns: [],
|
||||
isVisible: true,
|
||||
});
|
||||
|
||||
return userEntity;
|
||||
} catch (e) {
|
||||
Logger.error(e, 'Create new user');
|
||||
throw new InternalServerErrorException('Failed to register new user');
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
newAssetRepositoryMock,
|
||||
newCryptoRepositoryMock,
|
||||
newJobRepositoryMock,
|
||||
newLibraryRepositoryMock,
|
||||
newStorageRepositoryMock,
|
||||
newUserRepositoryMock,
|
||||
userStub,
|
||||
@@ -20,6 +21,7 @@ import { IAssetRepository } from '../asset';
|
||||
import { AuthUserDto } from '../auth';
|
||||
import { ICryptoRepository } from '../crypto';
|
||||
import { IJobRepository, JobName } from '../job';
|
||||
import { ILibraryRepository } from '../library';
|
||||
import { IStorageRepository } from '../storage';
|
||||
import { UpdateUserDto } from './dto/update-user.dto';
|
||||
import { UserResponseDto, mapUser } from './response-dto';
|
||||
@@ -129,6 +131,7 @@ describe(UserService.name, () => {
|
||||
let albumMock: jest.Mocked<IAlbumRepository>;
|
||||
let assetMock: jest.Mocked<IAssetRepository>;
|
||||
let jobMock: jest.Mocked<IJobRepository>;
|
||||
let libraryMock: jest.Mocked<ILibraryRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -136,10 +139,11 @@ describe(UserService.name, () => {
|
||||
albumMock = newAlbumRepositoryMock();
|
||||
assetMock = newAssetRepositoryMock();
|
||||
jobMock = newJobRepositoryMock();
|
||||
libraryMock = newLibraryRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
userMock = newUserRepositoryMock();
|
||||
|
||||
sut = new UserService(userMock, cryptoRepositoryMock, albumMock, assetMock, jobMock, storageMock);
|
||||
sut = new UserService(userMock, cryptoRepositoryMock, libraryMock, albumMock, assetMock, jobMock, storageMock);
|
||||
|
||||
when(userMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser);
|
||||
when(userMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser);
|
||||
|
||||
@@ -7,6 +7,7 @@ import { IAssetRepository } from '../asset/asset.repository';
|
||||
import { AuthUserDto } from '../auth';
|
||||
import { ICryptoRepository } from '../crypto/crypto.repository';
|
||||
import { IEntityJob, IJobRepository, JobName } from '../job';
|
||||
import { ILibraryRepository } from '../library/library.repository';
|
||||
import { StorageCore, StorageFolder } from '../storage';
|
||||
import { IStorageRepository } from '../storage/storage.repository';
|
||||
import { CreateUserDto, UpdateUserDto, UserCountDto } from './dto';
|
||||
@@ -30,13 +31,13 @@ export class UserService {
|
||||
constructor(
|
||||
@Inject(IUserRepository) private userRepository: IUserRepository,
|
||||
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
|
||||
|
||||
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
|
||||
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
) {
|
||||
this.userCore = new UserCore(userRepository, cryptoRepository);
|
||||
this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
|
||||
}
|
||||
|
||||
async getAll(authUser: AuthUserDto, isAll: boolean): Promise<UserResponseDto[]> {
|
||||
|
||||
Reference in New Issue
Block a user