feat(server): multi archive downloads (#956)

server/apps/immich/src/api-v1/album/album.controller.ts

@@ -27,6 +27,12 @@ import { AlbumResponseDto } from './response-dto/album-response.dto';
 import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
 import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
 import { Response as Res } from 'express';
+import {
+  IMMICH_ARCHIVE_COMPLETE,
+  IMMICH_ARCHIVE_FILE_COUNT,
+  IMMICH_CONTENT_LENGTH_HINT,
+} from '../../constants/download.constant';
+import { DownloadDto } from '../asset/dto/download-library.dto';
 
 // TODO might be worth creating a AlbumParamsDto that validates `albumId` instead of using the pipe.
 @Authenticated()
@@ -119,11 +125,18 @@ export class AlbumController {
   async downloadArchive(
     @GetAuthUser() authUser: AuthUserDto,
     @Param('albumId', new ParseUUIDPipe({ version: '4' })) albumId: string,
+    @Query(new ValidationPipe({ transform: true })) dto: DownloadDto,
     @Response({ passthrough: true }) res: Res,
   ): Promise<any> {
-    const { stream, filename, filesize } = await this.albumService.downloadArchive(authUser, albumId);
-    res.attachment(filename);
-    res.setHeader('X-Immich-Content-Length-Hint', filesize);
+    const { stream, fileName, fileSize, fileCount, complete } = await this.albumService.downloadArchive(
+      authUser,
+      albumId,
+      dto,
+    );
+    res.attachment(fileName);
+    res.setHeader(IMMICH_CONTENT_LENGTH_HINT, fileSize);
+    res.setHeader(IMMICH_ARCHIVE_FILE_COUNT, fileCount);
+    res.setHeader(IMMICH_ARCHIVE_COMPLETE, `${complete}`);
     return stream;
   }
 }

server/apps/immich/src/api-v1/album/album.module.ts

@@ -9,9 +9,13 @@ import { AssetAlbumEntity } from '@app/database/entities/asset-album.entity';
 import { UserAlbumEntity } from '@app/database/entities/user-album.entity';
 import { AlbumRepository, ALBUM_REPOSITORY } from './album-repository';
 import { AssetRepository, ASSET_REPOSITORY } from '../asset/asset-repository';
+import { DownloadModule } from '../../modules/download/download.module';
 
 @Module({
-  imports: [TypeOrmModule.forFeature([AssetEntity, UserEntity, AlbumEntity, AssetAlbumEntity, UserAlbumEntity])],
+  imports: [
+    TypeOrmModule.forFeature([AssetEntity, UserEntity, AlbumEntity, AssetAlbumEntity, UserAlbumEntity]),
+    DownloadModule,
+  ],
   controllers: [AlbumController],
   providers: [
     AlbumService,

server/apps/immich/src/api-v1/album/album.service.spec.ts

@@ -6,11 +6,13 @@ import { AlbumResponseDto } from './response-dto/album-response.dto';
 import { IAssetRepository } from '../asset/asset-repository';
 import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
 import { IAlbumRepository } from './album-repository';
+import { DownloadService } from '../../modules/download/download.service';
 
 describe('Album service', () => {
   let sut: AlbumService;
   let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
   let assetRepositoryMock: jest.Mocked<IAssetRepository>;
+  let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
 
   const authUser: AuthUserDto = Object.freeze({
     id: '1111',
@@ -142,7 +144,11 @@ describe('Album service', () => {
       getExistingAssets: jest.fn(),
     };
 
-    sut = new AlbumService(albumRepositoryMock, assetRepositoryMock);
+    downloadServiceMock = {
+      downloadArchive: jest.fn(),
+    };
+
+    sut = new AlbumService(albumRepositoryMock, assetRepositoryMock, downloadServiceMock as DownloadService);
   });
 
   it('creates album', async () => {

server/apps/immich/src/api-v1/album/album.service.ts

@@ -1,13 +1,4 @@
-import {
-  BadRequestException,
-  Inject,
-  Injectable,
-  NotFoundException,
-  ForbiddenException,
-  Logger,
-  InternalServerErrorException,
-  StreamableFile,
-} from '@nestjs/common';
+import { BadRequestException, Inject, Injectable, NotFoundException, ForbiddenException } from '@nestjs/common';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { CreateAlbumDto } from './dto/create-album.dto';
 import { AlbumEntity } from '@app/database/entities/album.entity';
@@ -21,14 +12,15 @@ import { AlbumCountResponseDto } from './response-dto/album-count-response.dto';
 import { ASSET_REPOSITORY, IAssetRepository } from '../asset/asset-repository';
 import { AddAssetsResponseDto } from './response-dto/add-assets-response.dto';
 import { AddAssetsDto } from './dto/add-assets.dto';
-import archiver from 'archiver';
-import { extname } from 'path';
+import { DownloadService } from '../../modules/download/download.service';
+import { DownloadDto } from '../asset/dto/download-library.dto';
 
 @Injectable()
 export class AlbumService {
   constructor(
     @Inject(ALBUM_REPOSITORY) private _albumRepository: IAlbumRepository,
     @Inject(ASSET_REPOSITORY) private _assetRepository: IAssetRepository,
+    private downloadService: DownloadService,
   ) {}
 
   private async _getAlbum({
@@ -162,35 +154,11 @@ export class AlbumService {
     return this._albumRepository.getCountByUserId(authUser.id);
   }
 
-  async downloadArchive(authUser: AuthUserDto, albumId: string) {
+  async downloadArchive(authUser: AuthUserDto, albumId: string, dto: DownloadDto) {
     const album = await this._getAlbum({ authUser, albumId, validateIsOwner: false });
-    if (!album.assets || album.assets.length === 0) {
-      throw new BadRequestException('Cannot download an empty album.');
-    }
+    const assets = (album.assets || []).map((asset) => asset.assetInfo).slice(dto.skip || 0);
 
-    try {
-      const archive = archiver('zip', { store: true });
-      const stream = new StreamableFile(archive);
-      let totalSize = 0;
-
-      for (const { assetInfo } of album.assets) {
-        const { originalPath } = assetInfo;
-        const name = `${assetInfo.exifInfo?.imageName || assetInfo.id}${extname(originalPath)}`;
-        archive.file(originalPath, { name });
-        totalSize += Number(assetInfo.exifInfo?.fileSizeInByte || 0);
-      }
-
-      archive.finalize();
-
-      return {
-        stream,
-        filename: `${album.albumName}.zip`,
-        filesize: totalSize,
-      };
-    } catch (e) {
-      Logger.error(`Error downloading album ${e}`, 'downloadArchive');
-      throw new InternalServerErrorException(`Failed to download album ${e}`, 'DownloadArchive');
-    }
+    return this.downloadService.downloadArchive(album.albumName, assets);
   }
 
   async _checkValidThumbnail(album: AlbumEntity) {

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -24,7 +24,7 @@ export interface IAssetRepository {
     checksum?: Buffer,
   ): Promise<AssetEntity>;
   update(asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
-  getAllByUserId(userId: string): Promise<AssetEntity[]>;
+  getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]>;
   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
   getById(assetId: string): Promise<AssetEntity>;
   getLocationsByUserId(userId: string): Promise<CuratedLocationsResponseDto[]>;
@@ -81,7 +81,7 @@ export class AssetRepository implements IAssetRepository {
 
   async getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto> {
     // Get asset count by AssetType
-    const res = await this.assetRepository
+    const items = await this.assetRepository
       .createQueryBuilder('asset')
       .select(`COUNT(asset.id)`, 'count')
       .addSelect(`asset.type`, 'type')
@@ -89,14 +89,24 @@ export class AssetRepository implements IAssetRepository {
       .groupBy('asset.type')
       .getRawMany();
 
-    const assetCountByUserId = new AssetCountByUserIdResponseDto(0, 0);
-    res.map((item) => {
-      if (item.type === 'IMAGE') {
-        assetCountByUserId.photos = item.count;
-      } else if (item.type === 'VIDEO') {
-        assetCountByUserId.videos = item.count;
-      }
-    });
+    const assetCountByUserId = new AssetCountByUserIdResponseDto();
+
+    // asset type to dto property mapping
+    const map: Record<AssetType, keyof AssetCountByUserIdResponseDto> = {
+      [AssetType.AUDIO]: 'audio',
+      [AssetType.IMAGE]: 'photos',
+      [AssetType.VIDEO]: 'videos',
+      [AssetType.OTHER]: 'other',
+    };
+
+    for (const item of items) {
+      const count = Number(item.count) || 0;
+      const assetType = item.type as AssetType;
+      const type = map[assetType];
+
+      assetCountByUserId[type] = count;
+      assetCountByUserId.total += count;
+    }
 
     return assetCountByUserId;
   }
@@ -207,12 +217,13 @@ export class AssetRepository implements IAssetRepository {
    * Get all assets belong to the user on the database
    * @param userId
    */
-  async getAllByUserId(userId: string): Promise<AssetEntity[]> {
+  async getAllByUserId(userId: string, skip?: number): Promise<AssetEntity[]> {
     const query = this.assetRepository
       .createQueryBuilder('asset')
       .where('asset.userId = :userId', { userId: userId })
       .andWhere('asset.resizePath is not NULL')
       .leftJoinAndSelect('asset.exifInfo', 'exifInfo')
+      .skip(skip || 0)
       .orderBy('asset.createdAt', 'DESC');
 
     return await query.getMany();

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -52,6 +52,12 @@ import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-use
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
 import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
 import { UpdateAssetDto } from './dto/update-asset.dto';
+import { DownloadDto } from './dto/download-library.dto';
+import {
+  IMMICH_ARCHIVE_COMPLETE,
+  IMMICH_ARCHIVE_FILE_COUNT,
+  IMMICH_CONTENT_LENGTH_HINT,
+} from '../../constants/download.constant';
 
 @Authenticated()
 @ApiBearerAuth()
@@ -134,6 +140,20 @@ export class AssetController {
     return this.assetService.downloadFile(query, res);
   }
 
+  @Get('/download-library')
+  async downloadLibrary(
+    @GetAuthUser() authUser: AuthUserDto,
+    @Query(new ValidationPipe({ transform: true })) dto: DownloadDto,
+    @Response({ passthrough: true }) res: Res,
+  ): Promise<any> {
+    const { stream, fileName, fileSize, fileCount, complete } = await this.assetService.downloadLibrary(authUser, dto);
+    res.attachment(fileName);
+    res.setHeader(IMMICH_CONTENT_LENGTH_HINT, fileSize);
+    res.setHeader(IMMICH_ARCHIVE_FILE_COUNT, fileCount);
+    res.setHeader(IMMICH_ARCHIVE_COMPLETE, `${complete}`);
+    return stream;
+  }
+
   @Get('/file')
   async serveFile(
     @Headers() headers: Record<string, string>,

server/apps/immich/src/api-v1/asset/asset.module.ts

@@ -9,11 +9,13 @@ import { BackgroundTaskService } from '../../modules/background-task/background-
 import { CommunicationModule } from '../communication/communication.module';
 import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
 import { AssetRepository, ASSET_REPOSITORY } from './asset-repository';
+import { DownloadModule } from '../../modules/download/download.module';
 
 @Module({
   imports: [
     CommunicationModule,
     BackgroundTaskModule,
+    DownloadModule,
     TypeOrmModule.forFeature([AssetEntity]),
     BullModule.registerQueue({
       name: QueueNameEnum.ASSET_UPLOADED,

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -7,11 +7,13 @@ import { CreateAssetDto } from './dto/create-asset.dto';
 import { AssetCountByTimeBucket } from './response-dto/asset-count-by-time-group-response.dto';
 import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
+import { DownloadService } from '../../modules/download/download.service';
 
 describe('AssetService', () => {
   let sui: AssetService;
   let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
   let assetRepositoryMock: jest.Mocked<IAssetRepository>;
+  let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
 
   const authUser: AuthUserDto = Object.freeze({
     id: 'user_id_1',
@@ -89,7 +91,10 @@ describe('AssetService', () => {
   };
 
   const _getAssetCountByUserId = (): AssetCountByUserIdResponseDto => {
-    const result = new AssetCountByUserIdResponseDto(2, 2);
+    const result = new AssetCountByUserIdResponseDto();
+
+    result.videos = 2;
+    result.photos = 2;
 
     return result;
   };
@@ -114,7 +119,11 @@ describe('AssetService', () => {
       getExistingAssets: jest.fn(),
     };
 
-    sui = new AssetService(assetRepositoryMock, a);
+    downloadServiceMock = {
+      downloadArchive: jest.fn(),
+    };
+
+    sui = new AssetService(assetRepositoryMock, a, downloadServiceMock as DownloadService);
   });
 
   // Currently failing due to calculate checksum from a file

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -41,6 +41,8 @@ import { timeUtils } from '@app/common/utils';
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
 import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
 import { UpdateAssetDto } from './dto/update-asset.dto';
+import { DownloadService } from '../../modules/download/download.service';
+import { DownloadDto } from './dto/download-library.dto';
 
 const fileInfo = promisify(stat);
 
@@ -52,6 +54,8 @@ export class AssetService {
 
     @InjectRepository(AssetEntity)
     private assetRepository: Repository<AssetEntity>,
+
+    private downloadService: DownloadService,
   ) {}
 
   public async createUserAsset(
@@ -140,6 +144,12 @@ export class AssetService {
     return mapAsset(updatedAsset);
   }
 
+  public async downloadLibrary(user: AuthUserDto, dto: DownloadDto) {
+    const assets = await this._assetRepository.getAllByUserId(user.id, dto.skip);
+
+    return this.downloadService.downloadArchive(dto.name || `library`, assets);
+  }
+
   public async downloadFile(query: ServeFileDto, res: Res) {
     try {
       let fileReadStream = null;

server/apps/immich/src/api-v1/asset/dto/download-library.dto.ts

@@ -0,0 +1,14 @@
+import { Type } from 'class-transformer';
+import { IsNumber, IsOptional, IsPositive, IsString } from 'class-validator';
+
+export class DownloadDto {
+  @IsOptional()
+  @IsString()
+  name = '';
+
+  @IsOptional()
+  @IsPositive()
+  @IsNumber()
+  @Type(() => Number)
+  skip?: number;
+}

server/apps/immich/src/api-v1/asset/response-dto/asset-count-by-user-id-response.dto.ts

@@ -2,13 +2,17 @@ import { ApiProperty } from '@nestjs/swagger';
 
 export class AssetCountByUserIdResponseDto {
   @ApiProperty({ type: 'integer' })
-  photos!: number;
+  audio = 0;
 
   @ApiProperty({ type: 'integer' })
-  videos!: number;
+  photos = 0;
 
-  constructor(photos: number, videos: number) {
-    this.photos = photos;
-    this.videos = videos;
-  }
+  @ApiProperty({ type: 'integer' })
+  videos = 0;
+
+  @ApiProperty({ type: 'integer' })
+  other = 0;
+
+  @ApiProperty({ type: 'integer' })
+  total = 0;
 }

server/apps/immich/src/api-v1/server-info/server-info.service.ts

@@ -9,6 +9,7 @@ import { Repository } from 'typeorm';
 import { InjectRepository } from '@nestjs/typeorm';
 import path from 'path';
 import { readdirSync, statSync } from 'fs';
+import { asHumanReadable } from '../../utils/human-readable.util';
 
 @Injectable()
 export class ServerInfoService {
@@ -23,9 +24,9 @@ export class ServerInfoService {
     const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2);
 
     const serverInfo = new ServerInfoResponseDto();
-    serverInfo.diskAvailable = ServerInfoService.getHumanReadableString(diskInfo.available);
-    serverInfo.diskSize = ServerInfoService.getHumanReadableString(diskInfo.total);
-    serverInfo.diskUse = ServerInfoService.getHumanReadableString(diskInfo.total - diskInfo.free);
+    serverInfo.diskAvailable = asHumanReadable(diskInfo.available);
+    serverInfo.diskSize = asHumanReadable(diskInfo.total);
+    serverInfo.diskUse = asHumanReadable(diskInfo.total - diskInfo.free);
     serverInfo.diskAvailableRaw = diskInfo.available;
     serverInfo.diskSizeRaw = diskInfo.total;
     serverInfo.diskUseRaw = diskInfo.total - diskInfo.free;
@@ -33,33 +34,6 @@ export class ServerInfoService {
     return serverInfo;
   }
 
-  private static getHumanReadableString(sizeInByte: number) {
-    const pepibyte = 1.126 * Math.pow(10, 15);
-    const tebibyte = 1.1 * Math.pow(10, 12);
-    const gibibyte = 1.074 * Math.pow(10, 9);
-    const mebibyte = 1.049 * Math.pow(10, 6);
-    const kibibyte = 1024;
-    // Pebibyte
-    if (sizeInByte >= pepibyte) {
-      // Pe
-      return `${(sizeInByte / pepibyte).toFixed(1)}PB`;
-    } else if (tebibyte <= sizeInByte && sizeInByte < pepibyte) {
-      // Te
-      return `${(sizeInByte / tebibyte).toFixed(1)}TB`;
-    } else if (gibibyte <= sizeInByte && sizeInByte < tebibyte) {
-      // Gi
-      return `${(sizeInByte / gibibyte).toFixed(1)}GB`;
-    } else if (mebibyte <= sizeInByte && sizeInByte < gibibyte) {
-      // Mega
-      return `${(sizeInByte / mebibyte).toFixed(1)}MB`;
-    } else if (kibibyte <= sizeInByte && sizeInByte < mebibyte) {
-      // Kibi
-      return `${(sizeInByte / kibibyte).toFixed(1)}KB`;
-    } else {
-      return `${sizeInByte}B`;
-    }
-  }
-
   async getStats(): Promise<ServerStatsResponseDto> {
     const res = await this.assetRepository
       .createQueryBuilder('asset')
@@ -90,11 +64,11 @@ export class ServerInfoService {
       const userDiskUsage = await ServerInfoService.getDirectoryStats(path.join(APP_UPLOAD_LOCATION, userId));
       usage.usageRaw = userDiskUsage.size;
       usage.objects = userDiskUsage.fileCount;
-      usage.usage = ServerInfoService.getHumanReadableString(usage.usageRaw);
+      usage.usage = asHumanReadable(usage.usageRaw);
       serverStats.usageRaw += usage.usageRaw;
       serverStats.objects += usage.objects;
     }
-    serverStats.usage = ServerInfoService.getHumanReadableString(serverStats.usageRaw);
+    serverStats.usage = asHumanReadable(serverStats.usageRaw);
     serverStats.usageByUser = Array.from(tmpMap.values());
     return serverStats;
   }

server/apps/immich/src/constants/download.constant.ts

@@ -0,0 +1,3 @@
+export const IMMICH_CONTENT_LENGTH_HINT = 'X-Immich-Content-Length-Hint';
+export const IMMICH_ARCHIVE_FILE_COUNT = 'X-Immich-Archive-File-Count';
+export const IMMICH_ARCHIVE_COMPLETE = 'X-Immich-Archive-Complete';

server/apps/immich/src/modules/download/download.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { DownloadService } from './download.service';
+
+@Module({
+  providers: [DownloadService],
+  exports: [DownloadService],
+})
+export class DownloadModule {}

server/apps/immich/src/modules/download/download.service.ts

@@ -0,0 +1,63 @@
+import { AssetEntity } from '@app/database/entities/asset.entity';
+import { BadRequestException, Injectable, InternalServerErrorException, Logger, StreamableFile } from '@nestjs/common';
+import archiver from 'archiver';
+import { extname } from 'path';
+import { asHumanReadable, HumanReadableSize } from '../../utils/human-readable.util';
+
+export interface DownloadArchive {
+  stream: StreamableFile;
+  fileName: string;
+  fileSize: number;
+  fileCount: number;
+  complete: boolean;
+}
+
+@Injectable()
+export class DownloadService {
+  private readonly logger = new Logger(DownloadService.name);
+
+  public async downloadArchive(name: string, assets: AssetEntity[]): Promise<DownloadArchive> {
+    if (!assets || assets.length === 0) {
+      throw new BadRequestException('No assets to download.');
+    }
+
+    try {
+      const archive = archiver('zip', { store: true });
+      const stream = new StreamableFile(archive);
+      let totalSize = 0;
+      let fileCount = 0;
+      let complete = true;
+
+      for (const { id, originalPath, exifInfo } of assets) {
+        const name = `${exifInfo?.imageName || id}${extname(originalPath)}`;
+        archive.file(originalPath, { name });
+        totalSize += Number(exifInfo?.fileSizeInByte || 0);
+        fileCount++;
+
+        // for easier testing, can be changed before merging.
+        if (totalSize > HumanReadableSize.GB * 20) {
+          complete = false;
+          this.logger.log(
+            `Archive size exceeded after ${fileCount} files, capping at ${totalSize} bytes (${asHumanReadable(
+              totalSize,
+            )})`,
+          );
+          break;
+        }
+      }
+
+      archive.finalize();
+
+      return {
+        stream,
+        fileName: `${name}.zip`,
+        fileSize: totalSize,
+        fileCount,
+        complete,
+      };
+    } catch (error) {
+      this.logger.error(`Error creating download archive ${error}`);
+      throw new InternalServerErrorException(`Failed to download ${name}: ${error}`, 'DownloadArchive');
+    }
+  }
+}

server/apps/immich/src/utils/human-readable.util.ts

@@ -0,0 +1,31 @@
+const KB = 1000;
+const MB = KB * 1000;
+const GB = MB * 1000;
+const TB = GB * 1000;
+const PB = TB * 1000;
+
+export const HumanReadableSize = { KB, MB, GB, TB, PB };
+
+export function asHumanReadable(bytes: number, precision = 1) {
+  if (bytes >= PB) {
+    return `${(bytes / PB).toFixed(precision)}PB`;
+  }
+
+  if (bytes >= TB) {
+    return `${(bytes / TB).toFixed(precision)}TB`;
+  }
+
+  if (bytes >= GB) {
+    return `${(bytes / GB).toFixed(precision)}GB`;
+  }
+
+  if (bytes >= MB) {
+    return `${(bytes / MB).toFixed(precision)}MB`;
+  }
+
+  if (bytes >= KB) {
+    return `${(bytes / KB).toFixed(precision)}KB`;
+  }
+
+  return `${bytes}B`;
+}