KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): pagination for asset queries in jobs (#2516)

Browse Source 
* feat(server): pagination for asset queries in jobs

* default mock value for getAll

* remove live photo name correction

* order paginated results by createdAt

* change log level

* move usePagination to domain
This commit is contained in:
Michel Heusschen
2023-05-22 20:05:06 +02:00
committed by GitHub
parent feadc45e75
commit f1384fea58
18 changed files with 253 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
server/libs/domain/src/search/search.service.ts
View File

@@ -7,8 +7,9 @@ import { mapAsset } from '../asset';
import { IAssetRepository } from '../asset/asset.repository';
import { AuthUserDto } from '../auth';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
import { usePagination } from '../domain.util';
import { AssetFaceId, IFaceRepository } from '../facial-recognition';
import { IAssetFaceJob, IBulkEntityJob, IJobRepository, JobName } from '../job';
import { IAssetFaceJob, IBulkEntityJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { IMachineLearningRepository } from '../smart-info';
import { SearchDto } from './dto';
import { SearchConfigResponseDto, SearchResponseDto } from './response-dto';
@@ -155,12 +156,15 @@ export class SearchService {
try {
// TODO: do this in batches based on searchIndexVersion
const assets = this.patchAssets(await this.assetRepository.getAll({ isVisible: true }));
this.logger.log(`Indexing ${assets.length} assets`);
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination, { isVisible: true }),
);
const chunkSize = 1000;
for (let i = 0; i < assets.length; i += chunkSize) {
await this.searchRepository.importAssets(assets.slice(i, i + chunkSize), false);
for await (const assets of assetPagination) {
this.logger.debug(`Indexing ${assets.length} assets`);
const patchedAssets = this.patchAssets(assets);
await this.searchRepository.importAssets(patchedAssets, false);
}
await this.searchRepository.importAssets([], true);