feat(server): asset entity audit (#3824)

* feat(server): audit log

* feedback

* Insert to database

* migration

* test

* controller/repository/service

* test

* module

* feat(server): implement audit endpoint

* directly return changed assets

* add daily cleanup of audit table

* fix tests

* review feedback

* ci

* refactor(server): audit implementation

* chore: open api

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/asset/asset.service.spec.ts

@@ -13,7 +13,7 @@ import {
 import { when } from 'jest-when';
 import { Readable } from 'stream';
 import { ICryptoRepository } from '../crypto';
-import { IJobRepository, JobName } from '../index';
+import { IJobRepository, JobName } from '../job';
 import { IStorageRepository } from '../storage';
 import { AssetStats, IAssetRepository } from './asset.repository';
 import { AssetService, UploadFieldName } from './asset.service';

server/src/domain/asset/asset.service.ts

@@ -16,18 +16,23 @@ import {
   AssetIdsDto,
   AssetJobName,
   AssetJobsDto,
+  AssetStatsDto,
   DownloadArchiveInfo,
   DownloadInfoDto,
   DownloadResponseDto,
+  MapMarkerDto,
+  mapStats,
   MemoryLaneDto,
   TimeBucketAssetDto,
   TimeBucketDto,
 } from './dto';
-import { AssetStatsDto, mapStats } from './dto/asset-statistics.dto';
-import { MapMarkerDto } from './dto/map-marker.dto';
-import { AssetResponseDto, mapAsset, MapMarkerResponseDto } from './response-dto';
-import { MemoryLaneResponseDto } from './response-dto/memory-lane-response.dto';
-import { TimeBucketResponseDto } from './response-dto/time-bucket-response.dto';
+import {
+  AssetResponseDto,
+  mapAsset,
+  MapMarkerResponseDto,
+  MemoryLaneResponseDto,
+  TimeBucketResponseDto,
+} from './response-dto';
 
 export enum UploadFieldName {
   ASSET_DATA = 'assetData',

server/src/domain/asset/response-dto/asset-response.dto.ts

@@ -84,3 +84,8 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto {
     checksum: entity.checksum.toString('base64'),
   };
 }
+
+export class MemoryLaneResponseDto {
+  title!: string;
+  assets!: AssetResponseDto[];
+}

server/src/domain/asset/response-dto/memory-lane-response.dto.ts

@@ -1,6 +0,0 @@
-import { AssetResponseDto } from './asset-response.dto';
-
-export class MemoryLaneResponseDto {
-  title!: string;
-  assets!: AssetResponseDto[];
-}

server/src/domain/audit/audi.service.spec.ts

@@ -0,0 +1,61 @@
+import { DatabaseAction, EntityType } from '@app/infra/entities';
+import { auditStub, authStub, IAccessRepositoryMock, newAccessRepositoryMock, newAuditRepositoryMock } from '@test';
+import { IAuditRepository } from './audit.repository';
+import { AuditService } from './audit.service';
+
+describe(AuditService.name, () => {
+  let sut: AuditService;
+  let accessMock: IAccessRepositoryMock;
+  let auditMock: jest.Mocked<IAuditRepository>;
+
+  beforeEach(async () => {
+    accessMock = newAccessRepositoryMock();
+    auditMock = newAuditRepositoryMock();
+    sut = new AuditService(accessMock, auditMock);
+  });
+
+  it('should work', () => {
+    expect(sut).toBeDefined();
+  });
+
+  describe('handleCleanup', () => {
+    it('should delete old audit entries', async () => {
+      await expect(sut.handleCleanup()).resolves.toBe(true);
+      expect(auditMock.removeBefore).toBeCalledWith(expect.any(Date));
+    });
+  });
+
+  describe('getDeletes', () => {
+    it('should require full sync if the request is older than 100 days', async () => {
+      auditMock.getAfter.mockResolvedValue([]);
+
+      const date = new Date(2022, 0, 1);
+      await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
+        needsFullSync: true,
+        ids: [],
+      });
+
+      expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
+        action: DatabaseAction.DELETE,
+        ownerId: authStub.admin.id,
+        entityType: EntityType.ASSET,
+      });
+    });
+
+    it('should get any new or updated assets and deleted ids', async () => {
+      auditMock.getAfter.mockResolvedValue([auditStub.delete]);
+
+      const date = new Date();
+      await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
+        needsFullSync: false,
+        ids: ['asset-deleted'],
+      });
+
+      expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
+        action: DatabaseAction.DELETE,
+        ownerId: authStub.admin.id,
+        entityType: EntityType.ASSET,
+      });
+    });
+  });
+});

server/src/domain/audit/audit.dto.ts

@@ -0,0 +1,24 @@
+import { EntityType } from '@app/infra/entities';
+import { ApiProperty } from '@nestjs/swagger';
+import { Type } from 'class-transformer';
+import { IsDate, IsEnum, IsOptional, IsUUID } from 'class-validator';
+
+export class AuditDeletesDto {
+  @IsDate()
+  @Type(() => Date)
+  after!: Date;
+
+  @ApiProperty({ enum: EntityType, enumName: 'EntityType' })
+  @IsEnum(EntityType)
+  entityType!: EntityType;
+
+  @IsOptional()
+  @IsUUID('4')
+  @ApiProperty({ format: 'uuid' })
+  userId?: string;
+}
+
+export class AuditDeletesResponseDto {
+  needsFullSync!: boolean;
+  ids!: string[];
+}

server/src/domain/audit/audit.repository.ts

@@ -0,0 +1,14 @@
+import { AuditEntity, DatabaseAction, EntityType } from '@app/infra/entities';
+
+export const IAuditRepository = 'IAuditRepository';
+
+export interface AuditSearch {
+  action?: DatabaseAction;
+  entityType?: EntityType;
+  ownerId?: string;
+}
+
+export interface IAuditRepository {
+  getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]>;
+  removeBefore(before: Date): Promise<void>;
+}

server/src/domain/audit/audit.service.ts

@@ -0,0 +1,43 @@
+import { DatabaseAction } from '@app/infra/entities';
+import { Inject, Injectable } from '@nestjs/common';
+import { DateTime } from 'luxon';
+import { AccessCore, IAccessRepository, Permission } from '../access';
+import { AuthUserDto } from '../auth';
+import { AUDIT_LOG_MAX_DURATION } from '../domain.constant';
+import { AuditDeletesDto, AuditDeletesResponseDto } from './audit.dto';
+import { IAuditRepository } from './audit.repository';
+
+@Injectable()
+export class AuditService {
+  private access: AccessCore;
+
+  constructor(
+    @Inject(IAccessRepository) accessRepository: IAccessRepository,
+    @Inject(IAuditRepository) private repository: IAuditRepository,
+  ) {
+    this.access = new AccessCore(accessRepository);
+  }
+
+  async handleCleanup(): Promise<boolean> {
+    await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
+    return true;
+  }
+
+  async getDeletes(authUser: AuthUserDto, dto: AuditDeletesDto): Promise<AuditDeletesResponseDto> {
+    const userId = dto.userId || authUser.id;
+    await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
+
+    const audits = await this.repository.getAfter(dto.after, {
+      ownerId: userId,
+      entityType: dto.entityType,
+      action: DatabaseAction.DELETE,
+    });
+
+    const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after));
+
+    return {
+      needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
+      ids: audits.map(({ entityId }) => entityId),
+    };
+  }
+}

server/src/domain/audit/index.ts

@@ -0,0 +1,3 @@
+export * from './audit.dto';
+export * from './audit.repository';
+export * from './audit.service';

server/src/domain/domain.constant.ts

@@ -1,8 +1,11 @@
 import { AssetType } from '@app/infra/entities';
 import { BadRequestException } from '@nestjs/common';
+import { Duration } from 'luxon';
 import { extname } from 'node:path';
 import pkg from 'src/../../package.json';
 
+export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
+
 const [major, minor, patch] = pkg.version.split('.');
 
 export interface IServerVersion {

server/src/domain/domain.module.ts

@@ -2,6 +2,7 @@ import { DynamicModule, Global, Module, ModuleMetadata, OnApplicationShutdown, P
 import { AlbumService } from './album';
 import { APIKeyService } from './api-key';
 import { AssetService } from './asset';
+import { AuditService } from './audit';
 import { AuthService } from './auth';
 import { FacialRecognitionService } from './facial-recognition';
 import { JobService } from './job';
@@ -23,6 +24,7 @@ const providers: Provider[] = [
   AlbumService,
   APIKeyService,
   AssetService,
+  AuditService,
   AuthService,
   FacialRecognitionService,
   JobService,

server/src/domain/index.ts

@@ -2,6 +2,7 @@ export * from './access';
 export * from './album';
 export * from './api-key';
 export * from './asset';
+export * from './audit';
 export * from './auth';
 export * from './communication';
 export * from './crypto';

server/src/domain/job/job.constants.ts

@@ -55,6 +55,7 @@ export enum JobName {
 
   // cleanup
   DELETE_FILES = 'delete-files',
+  CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
 
   // search
   SEARCH_INDEX_ASSETS = 'search-index-assets',
@@ -84,6 +85,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.USER_DELETE_CHECK]: QueueName.BACKGROUND_TASK,
   [JobName.USER_DELETION]: QueueName.BACKGROUND_TASK,
   [JobName.DELETE_FILES]: QueueName.BACKGROUND_TASK,
+  [JobName.CLEAN_OLD_AUDIT_LOGS]: QueueName.BACKGROUND_TASK,
   [JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK,
 
   // conversion

server/src/domain/job/job.repository.ts

@@ -68,6 +68,9 @@ export type JobItem =
   // Filesystem
   | { name: JobName.DELETE_FILES; data: IDeleteFilesJob }
 
+  // Audit log cleanup
+  | { name: JobName.CLEAN_OLD_AUDIT_LOGS; data?: IBaseJob }
+
   // Asset Deletion
   | { name: JobName.PERSON_CLEANUP; data?: IBaseJob }
 

server/src/domain/job/job.service.spec.ts

@@ -51,6 +51,7 @@ describe(JobService.name, () => {
         [{ name: JobName.USER_DELETE_CHECK }],
         [{ name: JobName.PERSON_CLEANUP }],
         [{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
+        [{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
       ]);
     });
   });

server/src/domain/job/job.service.ts

@@ -136,6 +136,7 @@ export class JobService {
     await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK });
     await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
     await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
+    await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
   }
 
   /**