KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-12-08 20:29:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(web,server)!: configure machine learning via the UI (#3768)

Browse Source
This commit is contained in:
Jason Rasmussen
2023-08-25 00:15:03 -04:00
committed by GitHub
parent 2cccef174a
commit 8211afb726
52 changed files with 831 additions and 649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
server/src/domain/smart-info/machine-learning.interface.ts
View File

@@ -20,8 +20,8 @@ export interface DetectFaceResult {
}
export interface IMachineLearningRepository {
classifyImage(input: MachineLearningInput): Promise<string[]>;
encodeImage(input: MachineLearningInput): Promise<number[]>;
encodeText(input: string): Promise<number[]>;
detectFaces(input: MachineLearningInput): Promise<DetectFaceResult[]>;
classifyImage(url: string, input: MachineLearningInput): Promise<string[]>;
encodeImage(url: string, input: MachineLearningInput): Promise<number[]>;
encodeText(url: string, input: string): Promise<number[]>;
detectFaces(url: string, input: MachineLearningInput): Promise<DetectFaceResult[]>;
}

14
server/src/domain/smart-info/smart-info.service.spec.ts
View File

@@ -5,9 +5,11 @@ import {
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newSmartInfoRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
import { IAssetRepository, WithoutProperty } from '../asset';
import { IJobRepository, JobName } from '../job';
import { ISystemConfigRepository } from '../system-config';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
import { SmartInfoService } from './smart-info.service';
@@ -20,16 +22,18 @@ const asset = {
describe(SmartInfoService.name, () => {
let sut: SmartInfoService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let smartMock: jest.Mocked<ISmartInfoRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
smartMock = newSmartInfoRepositoryMock();
jobMock = newJobRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
sut = new SmartInfoService(assetMock, jobMock, smartMock, machineMock);
sut = new SmartInfoService(assetMock, configMock, jobMock, smartMock, machineMock);
assetMock.getByIds.mockResolvedValue([asset]);
});
@@ -80,7 +84,9 @@ describe(SmartInfoService.name, () => {
await sut.handleClassifyImage({ id: asset.id });
expect(machineMock.classifyImage).toHaveBeenCalledWith({ imagePath: 'path/to/resize.ext' });
expect(machineMock.classifyImage).toHaveBeenCalledWith('http://immich-machine-learning:3003', {
imagePath: 'path/to/resize.ext',
});
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
tags: ['tag1', 'tag2', 'tag3'],
@@ -139,7 +145,9 @@ describe(SmartInfoService.name, () => {
await sut.handleEncodeClip({ id: asset.id });
expect(machineMock.encodeImage).toHaveBeenCalledWith({ imagePath: 'path/to/resize.ext' });
expect(machineMock.encodeImage).toHaveBeenCalledWith('http://immich-machine-learning:3003', {
imagePath: 'path/to/resize.ext',
});
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
clipEmbedding: [0.01, 0.02, 0.03],

41
server/src/domain/smart-info/smart-info.service.ts
View File

@@ -1,23 +1,31 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { IAssetRepository, WithoutProperty } from '../asset';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { ISystemConfigRepository, SystemConfigCore } from '../system-config';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
@Injectable()
export class SmartInfoService {
private logger = new Logger(SmartInfoService.name);
private configCore: SystemConfigCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private repository: ISmartInfoRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
) {}
) {
this.configCore = new SystemConfigCore(configRepository);
}
async handleQueueObjectTagging({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.tagImageEnabled) {
return true;
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
@@ -34,19 +42,28 @@ export class SmartInfoService {
}
async handleClassifyImage({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.tagImageEnabled) {
return true;
}
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset.resizePath) {
return false;
}
const tags = await this.machineLearning.classifyImage({ imagePath: asset.resizePath });
const tags = await this.machineLearning.classifyImage(machineLearning.url, { imagePath: asset.resizePath });
await this.repository.upsert({ assetId: asset.id, tags });
return true;
}
async handleQueueEncodeClip({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.clipEncodeEnabled) {
return true;
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
@@ -63,13 +80,17 @@ export class SmartInfoService {
}
async handleEncodeClip({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.clipEncodeEnabled) {
return true;
}
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset.resizePath) {
return false;
}
const clipEmbedding = await this.machineLearning.encodeImage({ imagePath: asset.resizePath });
const clipEmbedding = await this.machineLearning.encodeImage(machineLearning.url, { imagePath: asset.resizePath });
await this.repository.upsert({ assetId: asset.id, clipEmbedding: clipEmbedding });
return true;