KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-12-08 20:29:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): person thumbnail job (#4233)

Browse Source 
* refactor(server): person thumbnail job

* fix(server): set feature photo
This commit is contained in:
Jason Rasmussen
2023-09-26 03:03:22 -04:00
committed by GitHub
parent ea797c1723
commit 7bc6e9ef64
12 changed files with 153 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
server/src/domain/facial-recognition/facial-recognition.service.spec.ts
View File

@@ -26,45 +26,19 @@ import { FacialRecognitionService } from './facial-recognition.services';
const croppedFace = Buffer.from('Cropped Face');
const face = {
start: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 5,
y1: 5,
x2: 505,
y2: 505,
},
imageHeight: 1000,
imageWidth: 1000,
},
middle: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 100,
y1: 100,
x2: 200,
y2: 200,
},
imageHeight: 500,
imageWidth: 400,
embedding: [1, 2, 3, 4],
score: 0.2,
},
end: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 300,
y1: 300,
x2: 495,
y2: 495,
},
imageHeight: 500,
imageWidth: 500,
const detectFaceMock = {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 100,
y1: 100,
x2: 200,
y2: 200,
},
imageHeight: 500,
imageWidth: 400,
embedding: [1, 2, 3, 4],
score: 0.2,
};
const faceSearch = {
@@ -214,7 +188,7 @@ describe(FacialRecognitionService.name, () => {
});
it('should match existing people', async () => {
machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
@@ -233,7 +207,7 @@ describe(FacialRecognitionService.name, () => {
});
it('should create a new person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
personMock.create.mockResolvedValue(personStub.noName);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
@@ -253,60 +227,56 @@ describe(FacialRecognitionService.name, () => {
imageWidth: 400,
});
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.GENERATE_FACE_THUMBNAIL,
data: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 100,
y1: 100,
x2: 200,
y2: 200,
},
imageHeight: 500,
imageWidth: 400,
score: 0.2,
},
},
],
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
[{ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: 'person-1' } }],
]);
});
});
describe('handleGenerateFaceThumbnail', () => {
describe('handleGeneratePersonThumbnail', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleGenerateFaceThumbnail(face.middle)).resolves.toBe(true);
await expect(sut.handleGeneratePersonThumbnail({ id: 'person-1' })).resolves.toBe(true);
expect(assetMock.getByIds).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
});
it('should skip an asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
await sut.handleGenerateFaceThumbnail(face.middle);
it('should skip a person not found', async () => {
personMock.getById.mockResolvedValue(null);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).not.toHaveBeenCalled();
});
it('should skip an asset without a thumbnail', async () => {
it('should skip a person without a face asset id', async () => {
personMock.getById.mockResolvedValue(personStub.noThumbnail);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).not.toHaveBeenCalled();
});
it('should skip an person with a face asset id not found', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.middle.assetId });
faceMock.getByIds.mockResolvedValue([faceStub.face1]);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).not.toHaveBeenCalled();
});
it('should skip a person with a face asset id without a thumbnail', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.middle.assetId });
faceMock.getByIds.mockResolvedValue([faceStub.face1]);
assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
await sut.handleGenerateFaceThumbnail(face.middle);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).not.toHaveBeenCalled();
});
it('should generate a thumbnail', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.middle.assetId });
faceMock.getByIds.mockResolvedValue([faceStub.middle]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateFaceThumbnail(face.middle);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
expect(assetMock.getByIds).toHaveBeenCalledWith([faceStub.middle.assetId]);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/pe/rs');
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 95,
@@ -321,16 +291,17 @@ describe(FacialRecognitionService.name, () => {
colorspace: Colorspace.P3,
});
expect(personMock.update).toHaveBeenCalledWith({
faceAssetId: 'asset-1',
id: 'person-1',
thumbnailPath: 'upload/thumbs/user-id/pe/rs/person-1.jpeg',
});
});
it('should generate a thumbnail without going negative', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.start.assetId });
faceMock.getByIds.mockResolvedValue([faceStub.start]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateFaceThumbnail(face.start);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 0,
@@ -347,9 +318,11 @@ describe(FacialRecognitionService.name, () => {
});
it('should generate a thumbnail without overflowing', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.end.assetId });
faceMock.getByIds.mockResolvedValue([faceStub.end]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateFaceThumbnail(face.end);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 297,

55
server/src/domain/facial-recognition/facial-recognition.services.ts
View File

@@ -1,7 +1,8 @@
import { PersonEntity } from '@app/infra/entities';
import { Inject, Logger } from '@nestjs/common';
import { IAssetRepository, WithoutProperty } from '../asset';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, IFaceThumbnailJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { CropOptions, FACE_THUMBNAIL_SIZE, IMediaRepository } from '../media';
import { IPersonRepository } from '../person/person.repository';
import { ISearchRepository } from '../search/search.repository';
@@ -89,18 +90,14 @@ export class FacialRecognitionService {
personId = faceSearchResult.items[0].personId;
}
let newPerson: PersonEntity | null = null;
if (!personId) {
this.logger.debug('No matches, creating a new person.');
const person = await this.personRepository.create({ ownerId: asset.ownerId });
personId = person.id;
await this.jobRepository.queue({
name: JobName.GENERATE_FACE_THUMBNAIL,
data: { assetId: asset.id, personId, ...rest },
});
newPerson = await this.personRepository.create({ ownerId: asset.ownerId });
personId = newPerson.id;
}
const faceId: AssetFaceId = { assetId: asset.id, personId };
await this.faceRepository.create({
...faceId,
embedding,
@@ -112,6 +109,11 @@ export class FacialRecognitionService {
boundingBoxY2: rest.boundingBox.y2,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
if (newPerson) {
await this.personRepository.update({ id: personId, faceAssetId: asset.id });
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
}
}
return true;
@@ -132,24 +134,41 @@ export class FacialRecognitionService {
return true;
}
async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) {
const { machineLearning } = await this.configCore.getConfig();
async handleGeneratePersonThumbnail(data: IEntityJob) {
const { machineLearning, thumbnail } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
return true;
}
const { assetId, personId, boundingBox, imageWidth, imageHeight } = data;
const person = await this.personRepository.getById(data.id);
if (!person?.faceAssetId) {
return false;
}
const [face] = await this.faceRepository.getByIds([{ personId: person.id, assetId: person.faceAssetId }]);
if (!face) {
return false;
}
const {
assetId,
personId,
boundingBoxX1: x1,
boundingBoxX2: x2,
boundingBoxY1: y1,
boundingBoxY2: y2,
imageWidth,
imageHeight,
} = face;
const [asset] = await this.assetRepository.getByIds([assetId]);
if (!asset || !asset.resizePath) {
if (!asset?.resizePath) {
return false;
}
this.logger.verbose(`Cropping face for person: ${personId}`);
const output = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`);
const { x1, y1, x2, y2 } = boundingBox;
const thumbnailPath = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`);
const halfWidth = (x2 - x1) / 2;
const halfHeight = (y2 - y1) / 2;
@@ -175,7 +194,6 @@ export class FacialRecognitionService {
height: newHalfSize * 2,
};
const { thumbnail } = await this.configCore.getConfig();
const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions);
const thumbnailOptions = {
format: 'jpeg',
@@ -183,8 +201,9 @@ export class FacialRecognitionService {
colorspace: thumbnail.colorspace,
quality: thumbnail.quality,
} as const;
await this.mediaRepository.resize(croppedOutput, output, thumbnailOptions);
await this.personRepository.update({ id: personId, thumbnailPath: output, faceAssetId: data.assetId });
await this.mediaRepository.resize(croppedOutput, thumbnailPath, thumbnailOptions);
await this.personRepository.update({ id: personId, thumbnailPath });
return true;
}