feat(server,web): libraries (#3124)

* feat: libraries

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex <alex.tran1502@gmail.com>

server/src/infra/repositories/access.repository.ts

@@ -1,7 +1,7 @@
 import { IAccessRepository } from '@app/domain';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
-import { AlbumEntity, AssetEntity, PartnerEntity, PersonEntity, SharedLinkEntity } from '../entities';
+import { AlbumEntity, AssetEntity, LibraryEntity, PartnerEntity, PersonEntity, SharedLinkEntity } from '../entities';
 
 export class AccessRepository implements IAccessRepository {
   constructor(
@@ -10,9 +10,29 @@ export class AccessRepository implements IAccessRepository {
     @InjectRepository(PartnerEntity) private partnerRepository: Repository<PartnerEntity>,
     @InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
     @InjectRepository(SharedLinkEntity) private sharedLinkRepository: Repository<SharedLinkEntity>,
+    @InjectRepository(LibraryEntity) private libraryRepository: Repository<LibraryEntity>,
   ) {}
 
   library = {
+    hasOwnerAccess: (userId: string, libraryId: string): Promise<boolean> => {
+      return this.libraryRepository.exist({
+        where: {
+          id: libraryId,
+          ownerId: userId,
+        },
+      });
+    },
+    hasPartnerAccess: (userId: string, partnerId: string): Promise<boolean> => {
+      return this.partnerRepository.exist({
+        where: {
+          sharedWithId: userId,
+          sharedById: partnerId,
+        },
+      });
+    },
+  };
+
+  timeline = {
     hasPartnerAccess: (userId: string, partnerId: string): Promise<boolean> => {
       return this.partnerRepository.exist({
         where: {

server/src/infra/repositories/asset.repository.ts

@@ -38,6 +38,12 @@ export class AssetRepository implements IAssetRepository {
     await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] });
   }
 
+  create(
+    asset: Omit<AssetEntity, 'id' | 'createdAt' | 'updatedAt' | 'ownerId' | 'livePhotoVideoId'>,
+  ): Promise<AssetEntity> {
+    return this.repository.save(asset);
+  }
+
   getByDate(ownerId: string, date: Date): Promise<AssetEntity[]> {
     // For reference of a correct approach although slower
 
@@ -85,6 +91,7 @@ export class AssetRepository implements IAssetRepository {
       },
     });
   }
+
   async deleteAll(ownerId: string): Promise<void> {
     await this.repository.delete({ ownerId });
   }
@@ -115,6 +122,39 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
+    return this.repository.find({
+      where: { library: { id: In(libraryIds) } },
+    });
+  }
+
+  getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null> {
+    return this.repository.findOne({
+      where: { library: { id: libraryId }, originalPath: originalPath },
+    });
+  }
+
+  getById(assetId: string): Promise<AssetEntity> {
+    return this.repository.findOneOrFail({
+      where: {
+        id: assetId,
+      },
+      relations: {
+        exifInfo: true,
+        tags: true,
+        sharedLinks: true,
+        smartInfo: true,
+        faces: {
+          person: true,
+        },
+      },
+    });
+  }
+
+  remove(asset: AssetEntity): Promise<AssetEntity> {
+    return this.repository.remove(asset);
+  }
+
   getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
     return paginate(this.repository, pagination, {
       where: {
@@ -273,13 +313,19 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
-  getWith(pagination: PaginationOptions, property: WithProperty): Paginated<AssetEntity> {
+  getWith(pagination: PaginationOptions, property: WithProperty, libraryId?: string): Paginated<AssetEntity> {
     let where: FindOptionsWhere<AssetEntity> | FindOptionsWhere<AssetEntity>[] = {};
 
     switch (property) {
       case WithProperty.SIDECAR:
         where = [{ sidecarPath: Not(IsNull()), isVisible: true }];
         break;
+      case WithProperty.IS_OFFLINE:
+        if (!libraryId) {
+          throw new Error('Library id is required when finding offline assets');
+        }
+        where = [{ isOffline: true, libraryId: libraryId }];
+        break;
 
       default:
         throw new Error(`Invalid getWith property: ${property}`);

server/src/infra/repositories/filesystem.provider.spec.ts

@@ -0,0 +1,209 @@
+import { CrawlOptionsDto } from '@app/domain';
+import mockfs from 'mock-fs';
+import { FilesystemProvider } from './filesystem.provider';
+
+describe(FilesystemProvider.name, () => {
+  const sut: FilesystemProvider = new FilesystemProvider();
+
+  describe('crawl', () => {
+    it('should return empty wnen crawling an empty path list', async () => {
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = [];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths).toHaveLength(0);
+    });
+
+    it('should crawl a single path', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+      });
+
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image.jpg'].sort());
+    });
+
+    it('should exclude by file extension', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/image.tif': '',
+      });
+
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      options.exclusionPatterns = ['**/*.tif'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image.jpg'].sort());
+    });
+
+    it('should exclude by file extension without case sensitivity', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/image.tif': '',
+      });
+
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      options.exclusionPatterns = ['**/*.TIF'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image.jpg'].sort());
+    });
+
+    it('should exclude by folder', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/raw/image.jpg': '',
+        '/photos/raw2/image.jpg': '',
+        '/photos/folder/raw/image.jpg': '',
+        '/photos/crawl/image.jpg': '',
+      });
+
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      options.exclusionPatterns = ['**/raw/**'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image.jpg', '/photos/raw2/image.jpg', '/photos/crawl/image.jpg'].sort());
+    });
+
+    it('should crawl multiple paths', async () => {
+      mockfs({
+        '/photos/image1.jpg': '',
+        '/images/image2.jpg': '',
+        '/albums/image3.jpg': '',
+      });
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/', '/images/', '/albums/'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image1.jpg', '/images/image2.jpg', '/albums/image3.jpg'].sort());
+    });
+
+    it('should support globbing paths', async () => {
+      mockfs({
+        '/photos1/image1.jpg': '',
+        '/photos2/image2.jpg': '',
+        '/images/image3.jpg': '',
+      });
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos*'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos1/image1.jpg', '/photos2/image2.jpg'].sort());
+    });
+
+    it('should crawl a single path without trailing slash', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+      });
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image.jpg'].sort());
+    });
+
+    // TODO: test for hidden paths (not yet implemented)
+
+    it('should crawl a single path', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/subfolder/image1.jpg': '',
+        '/photos/subfolder/image2.jpg': '',
+        '/image1.jpg': '',
+      });
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(
+        ['/photos/image.jpg', '/photos/subfolder/image1.jpg', '/photos/subfolder/image2.jpg'].sort(),
+      );
+    });
+
+    it('should filter file extensions', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/image.txt': '',
+        '/photos/1': '',
+      });
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(['/photos/image.jpg'].sort());
+    });
+
+    it('should include photo and video extensions', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/image.jpeg': '',
+        '/photos/image.heic': '',
+        '/photos/image.heif': '',
+        '/photos/image.png': '',
+        '/photos/image.gif': '',
+        '/photos/image.tif': '',
+        '/photos/image.tiff': '',
+        '/photos/image.webp': '',
+        '/photos/image.dng': '',
+        '/photos/image.nef': '',
+        '/videos/video.mp4': '',
+        '/videos/video.mov': '',
+        '/videos/video.webm': '',
+      });
+
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/', '/videos/'];
+      const paths: string[] = await sut.crawl(options);
+
+      expect(paths.sort()).toEqual(
+        [
+          '/photos/image.jpg',
+          '/photos/image.jpeg',
+          '/photos/image.heic',
+          '/photos/image.heif',
+          '/photos/image.png',
+          '/photos/image.gif',
+          '/photos/image.tif',
+          '/photos/image.tiff',
+          '/photos/image.webp',
+          '/photos/image.dng',
+          '/photos/image.nef',
+          '/videos/video.mp4',
+          '/videos/video.mov',
+          '/videos/video.webm',
+        ].sort(),
+      );
+    });
+
+    it('should check file extensions without case sensitivity', async () => {
+      mockfs({
+        '/photos/image.jpg': '',
+        '/photos/image.Jpg': '',
+        '/photos/image.jpG': '',
+        '/photos/image.JPG': '',
+        '/photos/image.jpEg': '',
+        '/photos/image.TIFF': '',
+        '/photos/image.tif': '',
+        '/photos/image.dng': '',
+        '/photos/image.NEF': '',
+      });
+
+      const options = new CrawlOptionsDto();
+      options.pathsToCrawl = ['/photos/'];
+      const paths: string[] = await sut.crawl(options);
+      expect(paths.sort()).toEqual(
+        [
+          '/photos/image.jpg',
+          '/photos/image.Jpg',
+          '/photos/image.jpG',
+          '/photos/image.JPG',
+          '/photos/image.jpEg',
+          '/photos/image.TIFF',
+          '/photos/image.tif',
+          '/photos/image.dng',
+          '/photos/image.NEF',
+        ].sort(),
+      );
+    });
+
+    afterEach(() => {
+      mockfs.restore();
+    });
+  });
+});

server/src/infra/repositories/filesystem.provider.ts

@@ -1,7 +1,15 @@
-import { DiskUsage, ImmichReadStream, ImmichZipStream, IStorageRepository } from '@app/domain';
+import {
+  CrawlOptionsDto,
+  DiskUsage,
+  ImmichReadStream,
+  ImmichZipStream,
+  IStorageRepository,
+  mimeTypes,
+} from '@app/domain';
 import archiver from 'archiver';
 import { constants, createReadStream, existsSync, mkdirSync } from 'fs';
 import fs, { readdir } from 'fs/promises';
+import { glob } from 'glob';
 import mv from 'mv';
 import { promisify } from 'node:util';
 import path from 'path';
@@ -52,6 +60,8 @@ export class FilesystemProvider implements IStorageRepository {
     await fs.unlink(file);
   }
 
+  stat = fs.stat;
+
   async unlinkDir(folder: string, options: { recursive?: boolean; force?: boolean }) {
     await fs.rm(folder, options);
   }
@@ -93,5 +103,25 @@ export class FilesystemProvider implements IStorageRepository {
     };
   }
 
+  async crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
+    const pathsToCrawl = crawlOptions.pathsToCrawl;
+
+    let paths: string;
+    if (!pathsToCrawl) {
+      // No paths to crawl, return empty list
+      return [];
+    } else if (pathsToCrawl.length === 1) {
+      paths = pathsToCrawl[0];
+    } else {
+      paths = '{' + pathsToCrawl.join(',') + '}';
+    }
+
+    paths = paths + '/**/*{' + mimeTypes.getSupportedFileExtensions().join(',') + '}';
+
+    return (await glob(paths, { nocase: true, nodir: true, ignore: crawlOptions.exclusionPatterns })).map((assetPath) =>
+      path.normalize(assetPath),
+    );
+  }
+
   readdir = readdir;
 }

server/src/infra/repositories/index.ts

@@ -9,6 +9,7 @@ export * from './face.repository';
 export * from './filesystem.provider';
 export * from './geocoding.repository';
 export * from './job.repository';
+export * from './library.repository';
 export * from './machine-learning.repository';
 export * from './media.repository';
 export * from './partner.repository';

server/src/infra/repositories/job.repository.ts

@@ -78,7 +78,7 @@ export class JobRepository implements IJobRepository {
     }
   }
 
-  private getQueue(queue: QueueName) {
+  private getQueue(queue: QueueName): Queue {
     return this.moduleRef.get<Queue>(getQueueToken(queue), { strict: false });
   }
 }

server/src/infra/repositories/library.repository.ts

@@ -0,0 +1,183 @@
+import { ILibraryRepository, LibraryStatsResponseDto } from '@app/domain';
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { IsNull, Not } from 'typeorm';
+import { Repository } from 'typeorm/repository/Repository';
+import { LibraryEntity, LibraryType } from '../entities';
+
+@Injectable()
+export class LibraryRepository implements ILibraryRepository {
+  constructor(@InjectRepository(LibraryEntity) private repository: Repository<LibraryEntity>) {}
+
+  get(id: string, withDeleted = false): Promise<LibraryEntity | null> {
+    return this.repository.findOneOrFail({
+      where: {
+        id,
+      },
+      relations: { owner: true },
+      withDeleted,
+    });
+  }
+
+  existsByName(name: string, withDeleted = false): Promise<boolean> {
+    return this.repository.exist({
+      where: {
+        name,
+      },
+      withDeleted,
+    });
+  }
+
+  getCountForUser(ownerId: string): Promise<number> {
+    return this.repository.countBy({ ownerId });
+  }
+
+  getDefaultUploadLibrary(ownerId: string): Promise<LibraryEntity | null> {
+    return this.repository.findOne({
+      where: {
+        ownerId: ownerId,
+        type: LibraryType.UPLOAD,
+      },
+      order: {
+        createdAt: 'ASC',
+      },
+    });
+  }
+
+  getUploadLibraryCount(ownerId: string): Promise<number> {
+    return this.repository.count({
+      where: {
+        ownerId: ownerId,
+        type: LibraryType.UPLOAD,
+      },
+    });
+  }
+
+  getAllByUserId(ownerId: string, type?: LibraryType): Promise<LibraryEntity[]> {
+    return this.repository.find({
+      where: {
+        ownerId,
+        isVisible: true,
+        type,
+      },
+      relations: {
+        owner: true,
+      },
+      order: {
+        createdAt: 'ASC',
+      },
+    });
+  }
+
+  getAll(withDeleted = false, type?: LibraryType): Promise<LibraryEntity[]> {
+    return this.repository.find({
+      where: { type },
+      relations: {
+        owner: true,
+      },
+      order: {
+        createdAt: 'ASC',
+      },
+      withDeleted,
+    });
+  }
+
+  getAllDeleted(): Promise<LibraryEntity[]> {
+    return this.repository.find({
+      where: {
+        isVisible: true,
+        deletedAt: Not(IsNull()),
+      },
+      relations: {
+        owner: true,
+      },
+      order: {
+        createdAt: 'ASC',
+      },
+      withDeleted: true,
+    });
+  }
+
+  create(library: Omit<LibraryEntity, 'id' | 'createdAt' | 'updatedAt' | 'ownerId'>): Promise<LibraryEntity> {
+    return this.repository.save(library);
+  }
+
+  async delete(id: string): Promise<void> {
+    await this.repository.delete({ id });
+  }
+
+  async softDelete(id: string): Promise<void> {
+    await this.repository.softDelete({ id });
+  }
+
+  async update(library: Partial<LibraryEntity>): Promise<LibraryEntity> {
+    return this.save(library);
+  }
+
+  async getStatistics(id: string): Promise<LibraryStatsResponseDto> {
+    const stats = await this.repository
+      .createQueryBuilder('libraries')
+      .addSelect(`COUNT(assets.id) FILTER (WHERE assets.type = 'IMAGE' AND assets.isVisible)`, 'photos')
+      .addSelect(`COUNT(assets.id) FILTER (WHERE assets.type = 'VIDEO' AND assets.isVisible)`, 'videos')
+      .addSelect('COALESCE(SUM(exif.fileSizeInByte), 0)', 'usage')
+      .leftJoin('libraries.assets', 'assets')
+      .leftJoin('assets.exifInfo', 'exif')
+      .groupBy('libraries.id')
+      .where('libraries.id = :id', { id })
+      .getRawOne();
+
+    return {
+      photos: Number(stats.photos),
+      videos: Number(stats.videos),
+      usage: Number(stats.usage),
+      total: Number(stats.photos) + Number(stats.videos),
+    };
+  }
+
+  async getOnlineAssetPaths(libraryId: string): Promise<string[]> {
+    // Return all non-offline asset paths for a given library
+    const rawResults = await this.repository
+      .createQueryBuilder('library')
+      .innerJoinAndSelect('library.assets', 'assets')
+      .where('library.id = :id', { id: libraryId })
+      .andWhere('assets.isOffline = false')
+      .select('assets.originalPath')
+      .getRawMany();
+
+    const results: string[] = [];
+
+    for (const rawPath of rawResults) {
+      results.push(rawPath.assets_originalPath);
+    }
+
+    return results;
+  }
+
+  async getAssetIds(libraryId: string, withDeleted = false): Promise<string[]> {
+    let query = await this.repository
+      .createQueryBuilder('library')
+      .innerJoinAndSelect('library.assets', 'assets')
+      .where('library.id = :id', { id: libraryId })
+      .select('assets.id');
+
+    if (withDeleted) {
+      query = query.withDeleted();
+    }
+
+    // Return all asset paths for a given library
+    const rawResults = await query.getRawMany();
+
+    const results: string[] = [];
+
+    for (const rawPath of rawResults) {
+      results.push(rawPath.assets_id);
+    }
+
+    return results;
+  }
+
+  private async save(library: Partial<LibraryEntity>) {
+    const { id } = await this.repository.save(library);
+    return this.repository.findOneByOrFail({ id });
+  }
+}