feat(server): pagination for asset queries in jobs (#2516)

* feat(server): pagination for asset queries in jobs

* default mock value for getAll

* remove live photo name correction

* order paginated results by createdAt

* change log level

* move usePagination to domain

server/libs/domain/src/storage-template/storage-template.service.spec.ts

@@ -36,7 +36,10 @@ describe(StorageTemplateService.name, () => {
 
   describe('handle template migration', () => {
     it('should handle no assets', async () => {
-      assetMock.getAll.mockResolvedValue([]);
+      assetMock.getAll.mockResolvedValue({
+        items: [],
+        hasNextPage: false,
+      });
       userMock.getList.mockResolvedValue([]);
 
       await sut.handleTemplateMigration();
@@ -45,7 +48,10 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should handle an asset with a duplicate destination', async () => {
-      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
       assetMock.save.mockResolvedValue(assetEntityStub.image);
       userMock.getList.mockResolvedValue([userEntityStub.user1]);
 
@@ -69,12 +75,15 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should skip when an asset already matches the template', async () => {
-      assetMock.getAll.mockResolvedValue([
-        {
-          ...assetEntityStub.image,
-          originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
-        },
-      ]);
+      assetMock.getAll.mockResolvedValue({
+        items: [
+          {
+            ...assetEntityStub.image,
+            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext',
+          },
+        ],
+        hasNextPage: false,
+      });
       userMock.getList.mockResolvedValue([userEntityStub.user1]);
 
       await sut.handleTemplateMigration();
@@ -86,12 +95,15 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should skip when an asset is probably a duplicate', async () => {
-      assetMock.getAll.mockResolvedValue([
-        {
-          ...assetEntityStub.image,
-          originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
-        },
-      ]);
+      assetMock.getAll.mockResolvedValue({
+        items: [
+          {
+            ...assetEntityStub.image,
+            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext',
+          },
+        ],
+        hasNextPage: false,
+      });
       userMock.getList.mockResolvedValue([userEntityStub.user1]);
 
       await sut.handleTemplateMigration();
@@ -103,7 +115,10 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should move an asset', async () => {
-      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
       assetMock.save.mockResolvedValue(assetEntityStub.image);
       userMock.getList.mockResolvedValue([userEntityStub.user1]);
 
@@ -121,7 +136,10 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should use the user storage label', async () => {
-      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
       assetMock.save.mockResolvedValue(assetEntityStub.image);
       userMock.getList.mockResolvedValue([userEntityStub.storageLabel]);
 
@@ -139,7 +157,10 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should not update the database if the move fails', async () => {
-      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
       storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
       userMock.getList.mockResolvedValue([userEntityStub.user1]);
 
@@ -154,7 +175,10 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should move the asset back if the database fails', async () => {
-      assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
+      assetMock.getAll.mockResolvedValue({
+        items: [assetEntityStub.image],
+        hasNextPage: false,
+      });
       assetMock.save.mockRejectedValue('Connection Error!');
       userMock.getList.mockResolvedValue([userEntityStub.user1]);
 
@@ -173,7 +197,6 @@ describe(StorageTemplateService.name, () => {
   });
 
   it('should handle an error', async () => {
-    assetMock.getAll.mockResolvedValue([]);
     storageMock.removeEmptyDirs.mockRejectedValue(new Error('Read only filesystem'));
     userMock.getList.mockResolvedValue([]);
 

server/libs/domain/src/storage-template/storage-template.service.ts

@@ -2,8 +2,8 @@ import { AssetEntity, SystemConfig } from '@app/infra/entities';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { IAssetRepository } from '../asset/asset.repository';
 import { APP_MEDIA_LOCATION } from '../domain.constant';
-import { getLivePhotoMotionFilename } from '../domain.util';
-import { IAssetJob } from '../job';
+import { getLivePhotoMotionFilename, usePagination } from '../domain.util';
+import { IAssetJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
 import { IStorageRepository } from '../storage/storage.repository';
 import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
 import { IUserRepository } from '../user/user.repository';
@@ -52,26 +52,21 @@ export class StorageTemplateService {
   async handleTemplateMigration() {
     try {
       console.time('migrating-time');
-      const assets = await this.assetRepository.getAll();
+
+      const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+        this.assetRepository.getAll(pagination),
+      );
       const users = await this.userRepository.getList();
 
-      const livePhotoMap: Record<string, AssetEntity> = {};
-
-      for (const asset of assets) {
-        if (asset.livePhotoVideoId) {
-          livePhotoMap[asset.livePhotoVideoId] = asset;
+      for await (const assets of assetPagination) {
+        for (const asset of assets) {
+          const user = users.find((user) => user.id === asset.ownerId);
+          const storageLabel = user?.storageLabel || null;
+          const filename = asset.originalFileName || asset.id;
+          await this.moveAsset(asset, { storageLabel, filename });
         }
       }
 
-      for (const asset of assets) {
-        const livePhotoParentAsset = livePhotoMap[asset.id];
-        // TODO: remove livePhoto specific stuff once upload is fixed
-        const user = users.find((user) => user.id === asset.ownerId);
-        const storageLabel = user?.storageLabel || null;
-        const filename = asset.originalFileName || livePhotoParentAsset?.originalFileName || asset.id;
-        await this.moveAsset(asset, { storageLabel, filename });
-      }
-
       this.logger.debug('Cleaning up empty directories...');
       await this.storageRepository.removeEmptyDirs(APP_MEDIA_LOCATION);
     } catch (error: any) {