mirror of
https://github.com/KevinMidboe/immich.git
synced 2026-02-12 03:09:20 +00:00
feat: facial recognition (#2180)
This commit is contained in:
Jason Rasmussen
GitHub
14
server/libs/domain/src/facial-recognition/face.repository.ts
Normal file
14
server/libs/domain/src/facial-recognition/face.repository.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { AssetFaceEntity } from '@app/infra/entities';
|
||||
|
||||
export const IFaceRepository = 'IFaceRepository';
|
||||
|
||||
export interface AssetFaceId {
|
||||
assetId: string;
|
||||
personId: string;
|
||||
}
|
||||
|
||||
export interface IFaceRepository {
|
||||
getAll(): Promise<AssetFaceEntity[]>;
|
||||
getByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]>;
|
||||
create(entity: Partial<AssetFaceEntity>): Promise<AssetFaceEntity>;
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
import {
|
||||
assetEntityStub,
|
||||
faceStub,
|
||||
newAssetRepositoryMock,
|
||||
newFaceRepositoryMock,
|
||||
newJobRepositoryMock,
|
||||
newMachineLearningRepositoryMock,
|
||||
newMediaRepositoryMock,
|
||||
newPersonRepositoryMock,
|
||||
newSearchRepositoryMock,
|
||||
newStorageRepositoryMock,
|
||||
personStub,
|
||||
} from '../../test';
|
||||
import { IAssetRepository, WithoutProperty } from '../asset';
|
||||
import { IJobRepository, JobName } from '../job';
|
||||
import { IMediaRepository } from '../media';
|
||||
import { IPersonRepository } from '../person';
|
||||
import { ISearchRepository } from '../search';
|
||||
import { IMachineLearningRepository } from '../smart-info';
|
||||
import { IStorageRepository } from '../storage';
|
||||
import { IFaceRepository } from './face.repository';
|
||||
import { FacialRecognitionService } from './facial-recognition.services';
|
||||
|
||||
const croppedFace = Buffer.from('Cropped Face');
|
||||
|
||||
const face = {
|
||||
start: {
|
||||
assetId: 'asset-1',
|
||||
personId: 'person-1',
|
||||
boundingBox: {
|
||||
x1: 5,
|
||||
y1: 5,
|
||||
x2: 505,
|
||||
y2: 505,
|
||||
},
|
||||
imageHeight: 1000,
|
||||
imageWidth: 1000,
|
||||
},
|
||||
middle: {
|
||||
assetId: 'asset-1',
|
||||
personId: 'person-1',
|
||||
boundingBox: {
|
||||
x1: 100,
|
||||
y1: 100,
|
||||
x2: 200,
|
||||
y2: 200,
|
||||
},
|
||||
imageHeight: 500,
|
||||
imageWidth: 400,
|
||||
embedding: [1, 2, 3, 4],
|
||||
score: 0.2,
|
||||
},
|
||||
end: {
|
||||
assetId: 'asset-1',
|
||||
personId: 'person-1',
|
||||
boundingBox: {
|
||||
x1: 300,
|
||||
y1: 300,
|
||||
x2: 495,
|
||||
y2: 495,
|
||||
},
|
||||
imageHeight: 500,
|
||||
imageWidth: 500,
|
||||
},
|
||||
};
|
||||
|
||||
const faceSearch = {
|
||||
noMatch: {
|
||||
total: 0,
|
||||
count: 0,
|
||||
page: 1,
|
||||
items: [],
|
||||
distances: [],
|
||||
facets: [],
|
||||
},
|
||||
oneMatch: {
|
||||
total: 1,
|
||||
count: 1,
|
||||
page: 1,
|
||||
items: [faceStub.face1],
|
||||
distances: [0.1],
|
||||
facets: [],
|
||||
},
|
||||
oneRemoteMatch: {
|
||||
total: 1,
|
||||
count: 1,
|
||||
page: 1,
|
||||
items: [faceStub.face1],
|
||||
distances: [0.8],
|
||||
facets: [],
|
||||
},
|
||||
};
|
||||
|
||||
describe(FacialRecognitionService.name, () => {
|
||||
let sut: FacialRecognitionService;
|
||||
let assetMock: jest.Mocked<IAssetRepository>;
|
||||
let faceMock: jest.Mocked<IFaceRepository>;
|
||||
let jobMock: jest.Mocked<IJobRepository>;
|
||||
let machineLearningMock: jest.Mocked<IMachineLearningRepository>;
|
||||
let mediaMock: jest.Mocked<IMediaRepository>;
|
||||
let personMock: jest.Mocked<IPersonRepository>;
|
||||
let searchMock: jest.Mocked<ISearchRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
|
||||
beforeEach(async () => {
|
||||
assetMock = newAssetRepositoryMock();
|
||||
faceMock = newFaceRepositoryMock();
|
||||
jobMock = newJobRepositoryMock();
|
||||
machineLearningMock = newMachineLearningRepositoryMock();
|
||||
mediaMock = newMediaRepositoryMock();
|
||||
personMock = newPersonRepositoryMock();
|
||||
searchMock = newSearchRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
|
||||
mediaMock.crop.mockResolvedValue(croppedFace);
|
||||
|
||||
sut = new FacialRecognitionService(
|
||||
assetMock,
|
||||
faceMock,
|
||||
jobMock,
|
||||
machineLearningMock,
|
||||
mediaMock,
|
||||
personMock,
|
||||
searchMock,
|
||||
storageMock,
|
||||
);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('handleQueueRecognizeFaces', () => {
|
||||
it('should queue missing assets', async () => {
|
||||
assetMock.getWithout.mockResolvedValue([assetEntityStub.image]);
|
||||
await sut.handleQueueRecognizeFaces({});
|
||||
|
||||
expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.FACES);
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({
|
||||
name: JobName.RECOGNIZE_FACES,
|
||||
data: { asset: assetEntityStub.image },
|
||||
});
|
||||
});
|
||||
|
||||
it('should queue all assets', async () => {
|
||||
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
|
||||
personMock.deleteAll.mockResolvedValue(5);
|
||||
searchMock.deleteAllFaces.mockResolvedValue(100);
|
||||
|
||||
await sut.handleQueueRecognizeFaces({ force: true });
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({
|
||||
name: JobName.RECOGNIZE_FACES,
|
||||
data: { asset: assetEntityStub.image },
|
||||
});
|
||||
});
|
||||
|
||||
it('should log an error', async () => {
|
||||
assetMock.getWithout.mockRejectedValue(new Error('Database unavailable'));
|
||||
await sut.handleQueueRecognizeFaces({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleRecognizeFaces', () => {
|
||||
it('should skip when no resize path', async () => {
|
||||
await sut.handleRecognizeFaces({ asset: assetEntityStub.noResizePath });
|
||||
expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle no results', async () => {
|
||||
machineLearningMock.detectFaces.mockResolvedValue([]);
|
||||
await sut.handleRecognizeFaces({ asset: assetEntityStub.image });
|
||||
expect(machineLearningMock.detectFaces).toHaveBeenCalledWith({
|
||||
thumbnailPath: assetEntityStub.image.resizePath,
|
||||
});
|
||||
expect(faceMock.create).not.toHaveBeenCalled();
|
||||
expect(jobMock.queue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should match existing people', async () => {
|
||||
machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
|
||||
searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
|
||||
|
||||
await sut.handleRecognizeFaces({ asset: assetEntityStub.image });
|
||||
|
||||
expect(faceMock.create).toHaveBeenCalledWith({
|
||||
personId: 'person-1',
|
||||
assetId: 'asset-id',
|
||||
embedding: [1, 2, 3, 4],
|
||||
});
|
||||
expect(jobMock.queue.mock.calls).toEqual([
|
||||
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
|
||||
[{ name: JobName.SEARCH_INDEX_ASSET, data: { ids: ['asset-id'] } }],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should create a new person', async () => {
|
||||
machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
|
||||
searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
|
||||
personMock.create.mockResolvedValue(personStub.noName);
|
||||
|
||||
await sut.handleRecognizeFaces({ asset: assetEntityStub.image });
|
||||
|
||||
expect(personMock.create).toHaveBeenCalledWith({ ownerId: assetEntityStub.image.ownerId });
|
||||
expect(faceMock.create).toHaveBeenCalledWith({
|
||||
personId: 'person-1',
|
||||
assetId: 'asset-id',
|
||||
embedding: [1, 2, 3, 4],
|
||||
});
|
||||
expect(jobMock.queue.mock.calls).toEqual([
|
||||
[
|
||||
{
|
||||
name: JobName.GENERATE_FACE_THUMBNAIL,
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
personId: 'person-1',
|
||||
boundingBox: {
|
||||
x1: 100,
|
||||
y1: 100,
|
||||
x2: 200,
|
||||
y2: 200,
|
||||
},
|
||||
imageHeight: 500,
|
||||
imageWidth: 400,
|
||||
score: 0.2,
|
||||
},
|
||||
},
|
||||
],
|
||||
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
|
||||
[{ name: JobName.SEARCH_INDEX_ASSET, data: { ids: ['asset-id'] } }],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should log an error', async () => {
|
||||
machineLearningMock.detectFaces.mockRejectedValue(new Error('machine learning unavailable'));
|
||||
await sut.handleRecognizeFaces({ asset: assetEntityStub.image });
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleGenerateFaceThumbnail', () => {
|
||||
it('should skip an asset not found', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([]);
|
||||
|
||||
await sut.handleGenerateFaceThumbnail(face.middle);
|
||||
|
||||
expect(mediaMock.crop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should skip an asset without a thumbnail', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]);
|
||||
|
||||
await sut.handleGenerateFaceThumbnail(face.middle);
|
||||
|
||||
expect(mediaMock.crop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should generate a thumbnail', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
|
||||
|
||||
await sut.handleGenerateFaceThumbnail(face.middle);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
|
||||
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
|
||||
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext', {
|
||||
left: 95,
|
||||
top: 95,
|
||||
width: 110,
|
||||
height: 110,
|
||||
});
|
||||
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
|
||||
format: 'jpeg',
|
||||
size: 250,
|
||||
});
|
||||
expect(personMock.update).toHaveBeenCalledWith({
|
||||
id: 'person-1',
|
||||
thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg',
|
||||
});
|
||||
});
|
||||
|
||||
it('should generate a thumbnail without going negative', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
|
||||
|
||||
await sut.handleGenerateFaceThumbnail(face.start);
|
||||
|
||||
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext', {
|
||||
left: 0,
|
||||
top: 0,
|
||||
width: 510,
|
||||
height: 510,
|
||||
});
|
||||
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
|
||||
format: 'jpeg',
|
||||
size: 250,
|
||||
});
|
||||
});
|
||||
|
||||
it('should generate a thumbnail without overflowing', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
|
||||
|
||||
await sut.handleGenerateFaceThumbnail(face.end);
|
||||
|
||||
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext', {
|
||||
left: 297,
|
||||
top: 297,
|
||||
width: 202,
|
||||
height: 202,
|
||||
});
|
||||
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
|
||||
format: 'jpeg',
|
||||
size: 250,
|
||||
});
|
||||
});
|
||||
|
||||
it('should log an error', async () => {
|
||||
assetMock.getByIds.mockRejectedValue(new Error('Database unavailable'));
|
||||
await sut.handleGenerateFaceThumbnail(face.middle);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
import { Inject, Logger } from '@nestjs/common';
|
||||
import { join } from 'path';
|
||||
import { IAssetRepository, WithoutProperty } from '../asset';
|
||||
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
|
||||
import { IAssetJob, IBaseJob, IFaceThumbnailJob, IJobRepository, JobName } from '../job';
|
||||
import { CropOptions, FACE_THUMBNAIL_SIZE, IMediaRepository } from '../media';
|
||||
import { IPersonRepository } from '../person/person.repository';
|
||||
import { ISearchRepository } from '../search/search.repository';
|
||||
import { IMachineLearningRepository } from '../smart-info';
|
||||
import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
|
||||
import { AssetFaceId, IFaceRepository } from './face.repository';
|
||||
|
||||
export class FacialRecognitionService {
|
||||
private logger = new Logger(FacialRecognitionService.name);
|
||||
private storageCore = new StorageCore();
|
||||
|
||||
constructor(
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@Inject(IFaceRepository) private faceRepository: IFaceRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
|
||||
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
|
||||
@Inject(IPersonRepository) private personRepository: IPersonRepository,
|
||||
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
) {}
|
||||
|
||||
async handleQueueRecognizeFaces({ force }: IBaseJob) {
|
||||
try {
|
||||
const assets = force
|
||||
? await this.assetRepository.getAll()
|
||||
: await this.assetRepository.getWithout(WithoutProperty.FACES);
|
||||
|
||||
if (force) {
|
||||
const people = await this.personRepository.deleteAll();
|
||||
const faces = await this.searchRepository.deleteAllFaces();
|
||||
this.logger.debug(`Deleted ${people} people and ${faces} faces`);
|
||||
}
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: { asset } });
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Unable to queue recognize faces`, error?.stack);
|
||||
}
|
||||
}
|
||||
|
||||
async handleRecognizeFaces(data: IAssetJob) {
|
||||
const { asset } = data;
|
||||
|
||||
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const faces = await this.machineLearning.detectFaces({ thumbnailPath: asset.resizePath });
|
||||
|
||||
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
|
||||
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` })));
|
||||
|
||||
for (const { embedding, ...rest } of faces) {
|
||||
const faceSearchResult = await this.searchRepository.searchFaces(embedding, { ownerId: asset.ownerId });
|
||||
|
||||
let personId: string | null = null;
|
||||
|
||||
// try to find a matching face and link to the associated person
|
||||
// The closer to 0, the better the match. Range is from 0 to 2
|
||||
if (faceSearchResult.total && faceSearchResult.distances[0] < 0.6) {
|
||||
this.logger.verbose(`Match face with distance ${faceSearchResult.distances[0]}`);
|
||||
personId = faceSearchResult.items[0].personId;
|
||||
}
|
||||
|
||||
if (!personId) {
|
||||
this.logger.debug('No matches, creating a new person.');
|
||||
const person = await this.personRepository.create({ ownerId: asset.ownerId });
|
||||
personId = person.id;
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.GENERATE_FACE_THUMBNAIL,
|
||||
data: { assetId: asset.id, personId, ...rest },
|
||||
});
|
||||
}
|
||||
|
||||
const faceId: AssetFaceId = { assetId: asset.id, personId };
|
||||
|
||||
await this.faceRepository.create({ ...faceId, embedding });
|
||||
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
|
||||
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [asset.id] } });
|
||||
}
|
||||
|
||||
// queue all faces for asset
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Unable run facial recognition pipeline: ${asset.id}`, error?.stack);
|
||||
}
|
||||
}
|
||||
|
||||
async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) {
|
||||
const { assetId, personId, boundingBox, imageWidth, imageHeight } = data;
|
||||
|
||||
try {
|
||||
const [asset] = await this.assetRepository.getByIds([assetId]);
|
||||
if (!asset || !asset.resizePath) {
|
||||
this.logger.warn(`Asset not found for facial cropping: ${assetId}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
this.logger.verbose(`Cropping face for person: ${personId}`);
|
||||
|
||||
const outputFolder = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
|
||||
const output = join(outputFolder, `${personId}.jpeg`);
|
||||
this.storageRepository.mkdirSync(outputFolder);
|
||||
|
||||
const { x1, y1, x2, y2 } = boundingBox;
|
||||
|
||||
const halfWidth = (x2 - x1) / 2;
|
||||
const halfHeight = (y2 - y1) / 2;
|
||||
|
||||
const middleX = Math.round(x1 + halfWidth);
|
||||
const middleY = Math.round(y1 + halfHeight);
|
||||
|
||||
// zoom out 10%
|
||||
const targetHalfSize = Math.floor(Math.max(halfWidth, halfHeight) * 1.1);
|
||||
|
||||
// get the longest distance from the center of the image without overflowing
|
||||
const newHalfSize = Math.min(
|
||||
middleX - Math.max(0, middleX - targetHalfSize),
|
||||
middleY - Math.max(0, middleY - targetHalfSize),
|
||||
Math.min(imageWidth - 1, middleX + targetHalfSize) - middleX,
|
||||
Math.min(imageHeight - 1, middleY + targetHalfSize) - middleY,
|
||||
);
|
||||
|
||||
const cropOptions: CropOptions = {
|
||||
left: middleX - newHalfSize,
|
||||
top: middleY - newHalfSize,
|
||||
width: newHalfSize * 2,
|
||||
height: newHalfSize * 2,
|
||||
};
|
||||
|
||||
const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions);
|
||||
await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' });
|
||||
await this.personRepository.update({ id: personId, thumbnailPath: output });
|
||||
} catch (error: Error | any) {
|
||||
this.logger.error(`Failed to crop face for asset: ${assetId}, person: ${personId} - ${error}`, error.stack);
|
||||
}
|
||||
}
|
||||
}
|
||||
2
server/libs/domain/src/facial-recognition/index.ts
Normal file
2
server/libs/domain/src/facial-recognition/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export * from './facial-recognition.services';
|
||||
export * from './face.repository';
|
||||
Reference in New Issue
Block a user