refactor(server)*: tsconfigs (#2689)

* refactor(server): tsconfigs

* chore: dummy commit

* fix: start.sh

* chore: restore original entry scripts
This commit is contained in:
Jason Rasmussen
2023-06-08 11:01:07 -04:00
committed by GitHub
parent a2130aa6c5
commit 8ebac41318
465 changed files with 209 additions and 332 deletions

View File

@@ -0,0 +1,5 @@
export * from './system-config-ffmpeg.dto';
export * from './system-config-oauth.dto';
export * from './system-config-password-login.dto';
export * from './system-config-storage-template.dto';
export * from './system-config.dto';

View File

@@ -0,0 +1,40 @@
import { IsEnum, IsString, IsInt, IsBoolean, Min, Max } from 'class-validator';
import { TranscodePreset } from '@app/infra/entities';
import { Type } from 'class-transformer';
import { ApiProperty } from '@nestjs/swagger';
export class SystemConfigFFmpegDto {
@IsInt()
@Min(0)
@Max(51)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
crf!: number;
@IsInt()
@Min(0)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
threads!: number;
@IsString()
preset!: string;
@IsString()
targetVideoCodec!: string;
@IsString()
targetAudioCodec!: string;
@IsString()
targetResolution!: string;
@IsString()
maxBitrate!: string;
@IsBoolean()
twoPass!: boolean;
@IsEnum(TranscodePreset)
transcode!: TranscodePreset;
}

View File

@@ -0,0 +1,73 @@
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator';
import { QueueName } from '../../job';
export class JobSettingsDto {
@IsInt()
@IsPositive()
@ApiProperty({ type: 'integer' })
concurrency!: number;
}
export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.THUMBNAIL_GENERATION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.METADATA_EXTRACTION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.VIDEO_CONVERSION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.OBJECT_TAGGING]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.CLIP_ENCODING]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.BACKGROUND_TASK]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.SEARCH]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.RECOGNIZE_FACES]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.SIDECAR]!: JobSettingsDto;
}

View File

@@ -0,0 +1,43 @@
import { IsBoolean, IsNotEmpty, IsString, IsUrl, ValidateIf } from 'class-validator';
const isEnabled = (config: SystemConfigOAuthDto) => config.enabled;
const isOverrideEnabled = (config: SystemConfigOAuthDto) => config.mobileOverrideEnabled;
export class SystemConfigOAuthDto {
@IsBoolean()
enabled!: boolean;
@ValidateIf(isEnabled)
@IsNotEmpty()
@IsString()
issuerUrl!: string;
@ValidateIf(isEnabled)
@IsNotEmpty()
@IsString()
clientId!: string;
@ValidateIf(isEnabled)
@IsNotEmpty()
@IsString()
clientSecret!: string;
@IsString()
scope!: string;
@IsString()
buttonText!: string;
@IsBoolean()
autoRegister!: boolean;
@IsBoolean()
autoLaunch!: boolean;
@IsBoolean()
mobileOverrideEnabled!: boolean;
@ValidateIf(isOverrideEnabled)
@IsUrl()
mobileRedirectUri!: string;
}

View File

@@ -0,0 +1,6 @@
import { IsBoolean } from 'class-validator';
export class SystemConfigPasswordLoginDto {
@IsBoolean()
enabled!: boolean;
}

View File

@@ -0,0 +1,7 @@
import { IsNotEmpty, IsString } from 'class-validator';
export class SystemConfigStorageTemplateDto {
@IsNotEmpty()
@IsString()
template!: string;
}

View File

@@ -0,0 +1,39 @@
import { SystemConfig } from '@app/infra/entities';
import { Type } from 'class-transformer';
import { IsObject, ValidateNested } from 'class-validator';
import { SystemConfigJobDto } from './system-config-job.dto';
import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
import { SystemConfigOAuthDto } from './system-config-oauth.dto';
import { SystemConfigPasswordLoginDto } from './system-config-password-login.dto';
import { SystemConfigStorageTemplateDto } from './system-config-storage-template.dto';
export class SystemConfigDto {
@Type(() => SystemConfigFFmpegDto)
@ValidateNested()
@IsObject()
ffmpeg!: SystemConfigFFmpegDto;
@Type(() => SystemConfigOAuthDto)
@ValidateNested()
@IsObject()
oauth!: SystemConfigOAuthDto;
@Type(() => SystemConfigPasswordLoginDto)
@ValidateNested()
@IsObject()
passwordLogin!: SystemConfigPasswordLoginDto;
@Type(() => SystemConfigStorageTemplateDto)
@ValidateNested()
@IsObject()
storageTemplate!: SystemConfigStorageTemplateDto;
@Type(() => SystemConfigJobDto)
@ValidateNested()
@IsObject()
job!: SystemConfigJobDto;
}
export function mapConfig(config: SystemConfig): SystemConfigDto {
return config;
}

View File

@@ -0,0 +1,5 @@
export * from './dto';
export * from './response-dto';
export * from './system-config.constants';
export * from './system-config.repository';
export * from './system-config.service';

View File

@@ -0,0 +1 @@
export * from './system-config-template-storage-option.dto';

View File

@@ -0,0 +1,9 @@
export class SystemConfigTemplateStorageOptionDto {
yearOptions!: string[];
monthOptions!: string[];
dayOptions!: string[];
hourOptions!: string[];
minuteOptions!: string[];
secondOptions!: string[];
presetOptions!: string[];
}

View File

@@ -0,0 +1,23 @@
export const supportedYearTokens = ['y', 'yy'];
export const supportedMonthTokens = ['M', 'MM', 'MMM', 'MMMM'];
export const supportedDayTokens = ['d', 'dd'];
export const supportedHourTokens = ['h', 'hh', 'H', 'HH'];
export const supportedMinuteTokens = ['m', 'mm'];
export const supportedSecondTokens = ['s', 'ss'];
export const supportedPresetTokens = [
'{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
'{{y}}/{{MM}}-{{dd}}/{{filename}}',
'{{y}}/{{MMMM}}-{{dd}}/{{filename}}',
'{{y}}/{{MM}}/{{filename}}',
'{{y}}/{{MMM}}/{{filename}}',
'{{y}}/{{MMMM}}/{{filename}}',
'{{y}}/{{MM}}/{{dd}}/{{filename}}',
'{{y}}/{{MMMM}}/{{dd}}/{{filename}}',
'{{y}}/{{y}}-{{MM}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
'{{y}}-{{MM}}-{{dd}}/{{filename}}',
'{{y}}-{{MMM}}-{{dd}}/{{filename}}',
'{{y}}-{{MMMM}}-{{dd}}/{{filename}}',
'{{y}}/{{y}}-{{MM}}/{{filename}}',
];
export const INITIAL_SYSTEM_CONFIG = 'INITIAL_SYSTEM_CONFIG';

View File

@@ -0,0 +1,139 @@
import {
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
SystemConfigValue,
TranscodePreset,
} from '@app/infra/entities';
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import * as _ from 'lodash';
import { Subject } from 'rxjs';
import { DeepPartial } from 'typeorm';
import { QueueName } from '../job/job.constants';
import { ISystemConfigRepository } from './system-config.repository';
export type SystemConfigValidator = (config: SystemConfig) => void | Promise<void>;
const defaults = Object.freeze<SystemConfig>({
ffmpeg: {
crf: 23,
threads: 0,
preset: 'ultrafast',
targetVideoCodec: 'h264',
targetAudioCodec: 'aac',
targetResolution: '720',
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.CLIP_ENCODING]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.OBJECT_TAGGING]: { concurrency: 2 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},
oauth: {
enabled: false,
issuerUrl: '',
clientId: '',
clientSecret: '',
mobileOverrideEnabled: false,
mobileRedirectUri: '',
scope: 'openid email profile',
buttonText: 'Login with OAuth',
autoRegister: true,
autoLaunch: false,
},
passwordLogin: {
enabled: true,
},
storageTemplate: {
template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
},
});
const singleton = new Subject<SystemConfig>();
@Injectable()
export class SystemConfigCore {
private logger = new Logger(SystemConfigCore.name);
private validators: SystemConfigValidator[] = [];
public config$ = singleton;
constructor(private repository: ISystemConfigRepository) {}
public getDefaults(): SystemConfig {
return defaults;
}
public addValidator(validator: SystemConfigValidator) {
this.validators.push(validator);
}
public async getConfig() {
const overrides = await this.repository.load();
const config: DeepPartial<SystemConfig> = {};
for (const { key, value } of overrides) {
// set via dot notation
_.set(config, key, value);
}
return _.defaultsDeep(config, defaults) as SystemConfig;
}
public async updateConfig(config: SystemConfig): Promise<SystemConfig> {
try {
for (const validator of this.validators) {
await validator(config);
}
} catch (e) {
this.logger.warn(`Unable to save system config due to a validation error: ${e}`);
throw new BadRequestException(e instanceof Error ? e.message : e);
}
const updates: SystemConfigEntity[] = [];
const deletes: SystemConfigEntity[] = [];
for (const key of Object.values(SystemConfigKey)) {
// get via dot notation
const item = { key, value: _.get(config, key) as SystemConfigValue };
const defaultValue = _.get(defaults, key);
const isMissing = !_.has(config, key);
if (isMissing || item.value === null || item.value === '' || item.value === defaultValue) {
deletes.push(item);
continue;
}
updates.push(item);
}
if (updates.length > 0) {
await this.repository.saveAll(updates);
}
if (deletes.length > 0) {
await this.repository.deleteKeys(deletes.map((item) => item.key));
}
const newConfig = await this.getConfig();
this.config$.next(newConfig);
return newConfig;
}
public async refreshConfig() {
const newConfig = await this.getConfig();
this.config$.next(newConfig);
}
}

View File

@@ -0,0 +1,9 @@
import { SystemConfigEntity } from '@app/infra/entities';
export const ISystemConfigRepository = 'ISystemConfigRepository';
export interface ISystemConfigRepository {
load(): Promise<SystemConfigEntity[]>;
saveAll(items: SystemConfigEntity[]): Promise<SystemConfigEntity[]>;
deleteKeys(keys: string[]): Promise<void>;
}

View File

@@ -0,0 +1,173 @@
import { SystemConfig, SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import { newJobRepositoryMock, newSystemConfigRepositoryMock, systemConfigStub } from '@test';
import { IJobRepository, JobName, QueueName } from '../job';
import { SystemConfigValidator } from './system-config.core';
import { ISystemConfigRepository } from './system-config.repository';
import { SystemConfigService } from './system-config.service';
const updates: SystemConfigEntity[] = [
{ key: SystemConfigKey.FFMPEG_CRF, value: 30 },
{ key: SystemConfigKey.OAUTH_AUTO_LAUNCH, value: true },
];
const updatedConfig = Object.freeze<SystemConfig>({
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.CLIP_ENCODING]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.OBJECT_TAGGING]: { concurrency: 2 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},
ffmpeg: {
crf: 30,
threads: 0,
preset: 'ultrafast',
targetAudioCodec: 'aac',
targetResolution: '720',
targetVideoCodec: 'h264',
maxBitrate: '0',
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
oauth: {
autoLaunch: true,
autoRegister: true,
buttonText: 'Login with OAuth',
clientId: '',
clientSecret: '',
enabled: false,
issuerUrl: '',
mobileOverrideEnabled: false,
mobileRedirectUri: '',
scope: 'openid email profile',
},
passwordLogin: {
enabled: true,
},
storageTemplate: {
template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
},
});
describe(SystemConfigService.name, () => {
let sut: SystemConfigService;
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
beforeEach(async () => {
configMock = newSystemConfigRepositoryMock();
jobMock = newJobRepositoryMock();
sut = new SystemConfigService(configMock, jobMock);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('getDefaults', () => {
it('should return the default config', () => {
configMock.load.mockResolvedValue(updates);
expect(sut.getDefaults()).toEqual(systemConfigStub.defaults);
expect(configMock.load).not.toHaveBeenCalled();
});
});
describe('addValidator', () => {
it('should call the validator on config changes', async () => {
const validator: SystemConfigValidator = jest.fn();
sut.addValidator(validator);
await sut.updateConfig(systemConfigStub.defaults);
expect(validator).toHaveBeenCalledWith(systemConfigStub.defaults);
});
});
describe('getConfig', () => {
it('should return the default config', async () => {
configMock.load.mockResolvedValue([]);
await expect(sut.getConfig()).resolves.toEqual(systemConfigStub.defaults);
});
it('should merge the overrides', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_CRF, value: 30 },
{ key: SystemConfigKey.OAUTH_AUTO_LAUNCH, value: true },
]);
await expect(sut.getConfig()).resolves.toEqual(updatedConfig);
});
});
describe('getStorageTemplateOptions', () => {
it('should send back the datetime variables', () => {
expect(sut.getStorageTemplateOptions()).toEqual({
dayOptions: ['d', 'dd'],
hourOptions: ['h', 'hh', 'H', 'HH'],
minuteOptions: ['m', 'mm'],
monthOptions: ['M', 'MM', 'MMM', 'MMMM'],
presetOptions: [
'{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
'{{y}}/{{MM}}-{{dd}}/{{filename}}',
'{{y}}/{{MMMM}}-{{dd}}/{{filename}}',
'{{y}}/{{MM}}/{{filename}}',
'{{y}}/{{MMM}}/{{filename}}',
'{{y}}/{{MMMM}}/{{filename}}',
'{{y}}/{{MM}}/{{dd}}/{{filename}}',
'{{y}}/{{MMMM}}/{{dd}}/{{filename}}',
'{{y}}/{{y}}-{{MM}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
'{{y}}-{{MM}}-{{dd}}/{{filename}}',
'{{y}}-{{MMM}}-{{dd}}/{{filename}}',
'{{y}}-{{MMMM}}-{{dd}}/{{filename}}',
'{{y}}/{{y}}-{{MM}}/{{filename}}',
],
secondOptions: ['s', 'ss'],
yearOptions: ['y', 'yy'],
});
});
});
describe('updateConfig', () => {
it('should notify the microservices process', async () => {
configMock.load.mockResolvedValue(updates);
await expect(sut.updateConfig(updatedConfig)).resolves.toEqual(updatedConfig);
expect(configMock.saveAll).toHaveBeenCalledWith(updates);
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.SYSTEM_CONFIG_CHANGE });
});
it('should throw an error if the config is not valid', async () => {
const validator = jest.fn().mockRejectedValue('invalid config');
sut.addValidator(validator);
await expect(sut.updateConfig(updatedConfig)).rejects.toBeInstanceOf(BadRequestException);
expect(validator).toHaveBeenCalledWith(updatedConfig);
expect(configMock.saveAll).not.toHaveBeenCalled();
});
});
describe('refreshConfig', () => {
it('should notify the subscribers', async () => {
const changeMock = jest.fn();
const subscription = sut.config$.subscribe(changeMock);
await sut.refreshConfig();
expect(changeMock).toHaveBeenCalledWith(systemConfigStub.defaults);
subscription.unsubscribe();
});
});
});

View File

@@ -0,0 +1,69 @@
import { ISystemConfigRepository } from '.';
import {
supportedDayTokens,
supportedHourTokens,
supportedMinuteTokens,
supportedMonthTokens,
supportedPresetTokens,
supportedSecondTokens,
supportedYearTokens,
} from './system-config.constants';
import { Inject, Injectable } from '@nestjs/common';
import { IJobRepository, JobName } from '../job';
import { mapConfig, SystemConfigDto } from './dto/system-config.dto';
import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
import { SystemConfigCore, SystemConfigValidator } from './system-config.core';
@Injectable()
export class SystemConfigService {
private core: SystemConfigCore;
constructor(
@Inject(ISystemConfigRepository) repository: ISystemConfigRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
this.core = new SystemConfigCore(repository);
}
get config$() {
return this.core.config$;
}
async getConfig(): Promise<SystemConfigDto> {
const config = await this.core.getConfig();
return mapConfig(config);
}
getDefaults(): SystemConfigDto {
const config = this.core.getDefaults();
return mapConfig(config);
}
async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
const config = await this.core.updateConfig(dto);
await this.jobRepository.queue({ name: JobName.SYSTEM_CONFIG_CHANGE });
return mapConfig(config);
}
async refreshConfig() {
await this.core.refreshConfig();
return true;
}
addValidator(validator: SystemConfigValidator) {
this.core.addValidator(validator);
}
getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto {
const options = new SystemConfigTemplateStorageOptionDto();
options.dayOptions = supportedDayTokens;
options.monthOptions = supportedMonthTokens;
options.yearOptions = supportedYearTokens;
options.hourOptions = supportedHourTokens;
options.secondOptions = supportedSecondTokens;
options.minuteOptions = supportedMinuteTokens;
options.presetOptions = supportedPresetTokens;
return options;
}
}