refactor(server): jobs and processors (#1787)

* refactor: jobs and processors

* refactor: storage migration processor

* fix: tests

* fix: code warning

* chore: ignore coverage from infra

* fix: sync move asset logic between job core and asset core

* refactor: move error handling inside of catch

* refactor(server): job core into dedicated service calls

* refactor: smart info

* fix: tests

* chore: smart info tests

* refactor: use asset repository

* refactor: thumbnail processor

* chore: coverage reqs

server/libs/domain/src/user/user.repository.ts

@@ -11,9 +11,10 @@ export interface IUserRepository {
   getAdmin(): Promise<UserEntity | null>;
   getByEmail(email: string, withPassword?: boolean): Promise<UserEntity | null>;
   getByOAuthId(oauthId: string): Promise<UserEntity | null>;
+  getDeletedUsers(): Promise<UserEntity[]>;
   getList(filter?: UserListFilter): Promise<UserEntity[]>;
   create(user: Partial<UserEntity>): Promise<UserEntity>;
   update(id: string, user: Partial<UserEntity>): Promise<UserEntity>;
-  delete(user: UserEntity): Promise<UserEntity>;
+  delete(user: UserEntity, hard?: boolean): Promise<UserEntity>;
   restore(user: UserEntity): Promise<UserEntity>;
 }

server/libs/domain/src/user/user.service.spec.ts

@@ -1,13 +1,34 @@
-import { IUserRepository } from './user.repository';
 import { UserEntity } from '@app/infra/db/entities';
 import { BadRequestException, ForbiddenException, NotFoundException } from '@nestjs/common';
 import { when } from 'jest-when';
-import { newCryptoRepositoryMock, newUserRepositoryMock } from '../../test';
+import {
+  newAlbumRepositoryMock,
+  newAssetRepositoryMock,
+  newCryptoRepositoryMock,
+  newJobRepositoryMock,
+  newKeyRepositoryMock,
+  newStorageRepositoryMock,
+  newUserRepositoryMock,
+  newUserTokenRepositoryMock,
+} from '../../test';
+import { IAlbumRepository } from '../album';
+import { IKeyRepository } from '../api-key';
+import { IAssetRepository } from '../asset';
 import { AuthUserDto } from '../auth';
 import { ICryptoRepository } from '../crypto';
+import { IJobRepository, JobName } from '../job';
+import { IStorageRepository } from '../storage';
+import { IUserTokenRepository } from '../user-token';
 import { UpdateUserDto } from './dto/update-user.dto';
+import { IUserRepository } from './user.repository';
 import { UserService } from './user.service';
 
+const makeDeletedAt = (daysAgo: number) => {
+  const deletedAt = new Date();
+  deletedAt.setDate(deletedAt.getDate() - daysAgo);
+  return deletedAt;
+};
+
 const adminUserAuth: AuthUserDto = Object.freeze({
   id: 'admin_id',
   email: 'admin@test.com',
@@ -83,10 +104,35 @@ describe(UserService.name, () => {
   let userRepositoryMock: jest.Mocked<IUserRepository>;
   let cryptoRepositoryMock: jest.Mocked<ICryptoRepository>;
 
+  let albumMock: jest.Mocked<IAlbumRepository>;
+  let assetMock: jest.Mocked<IAssetRepository>;
+  let jobMock: jest.Mocked<IJobRepository>;
+  let keyMock: jest.Mocked<IKeyRepository>;
+  let storageMock: jest.Mocked<IStorageRepository>;
+  let tokenMock: jest.Mocked<IUserTokenRepository>;
+
   beforeEach(async () => {
     userRepositoryMock = newUserRepositoryMock();
     cryptoRepositoryMock = newCryptoRepositoryMock();
-    sut = new UserService(userRepositoryMock, cryptoRepositoryMock);
+
+    albumMock = newAlbumRepositoryMock();
+    assetMock = newAssetRepositoryMock();
+    jobMock = newJobRepositoryMock();
+    keyMock = newKeyRepositoryMock();
+    storageMock = newStorageRepositoryMock();
+    tokenMock = newUserTokenRepositoryMock();
+    userRepositoryMock = newUserRepositoryMock();
+
+    sut = new UserService(
+      userRepositoryMock,
+      cryptoRepositoryMock,
+      albumMock,
+      assetMock,
+      jobMock,
+      keyMock,
+      storageMock,
+      tokenMock,
+    );
 
     when(userRepositoryMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser);
     when(userRepositoryMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser);
@@ -374,4 +420,64 @@ describe(UserService.name, () => {
       expect(update.password).toBeDefined();
     });
   });
+
+  describe('handleUserDeleteCheck', () => {
+    it('should skip users not ready for deletion', async () => {
+      userRepositoryMock.getDeletedUsers.mockResolvedValue([
+        {},
+        { deletedAt: undefined },
+        { deletedAt: null },
+        { deletedAt: makeDeletedAt(5) },
+      ] as UserEntity[]);
+
+      await sut.handleUserDeleteCheck();
+
+      expect(userRepositoryMock.getDeletedUsers).toHaveBeenCalled();
+      expect(jobMock.queue).not.toHaveBeenCalled();
+    });
+
+    it('should queue user ready for deletion', async () => {
+      const user = { deletedAt: makeDeletedAt(10) };
+      userRepositoryMock.getDeletedUsers.mockResolvedValue([user] as UserEntity[]);
+
+      await sut.handleUserDeleteCheck();
+
+      expect(userRepositoryMock.getDeletedUsers).toHaveBeenCalled();
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.USER_DELETION, data: { user } });
+    });
+  });
+
+  describe('handleUserDelete', () => {
+    it('should skip users not ready for deletion', async () => {
+      const user = { deletedAt: makeDeletedAt(5) } as UserEntity;
+
+      await sut.handleUserDelete({ user });
+
+      expect(storageMock.unlinkDir).not.toHaveBeenCalled();
+      expect(userRepositoryMock.delete).not.toHaveBeenCalled();
+    });
+
+    it('should delete the user and associated assets', async () => {
+      const user = { id: 'deleted-user', deletedAt: makeDeletedAt(10) } as UserEntity;
+
+      await sut.handleUserDelete({ user });
+
+      expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/deleted-user', { force: true, recursive: true });
+      expect(tokenMock.deleteAll).toHaveBeenCalledWith(user.id);
+      expect(keyMock.deleteAll).toHaveBeenCalledWith(user.id);
+      expect(albumMock.deleteAll).toHaveBeenCalledWith(user.id);
+      expect(assetMock.deleteAll).toHaveBeenCalledWith(user.id);
+      expect(userRepositoryMock.delete).toHaveBeenCalledWith(user, true);
+    });
+
+    it('should handle an error', async () => {
+      const user = { id: 'deleted-user', deletedAt: makeDeletedAt(10) } as UserEntity;
+
+      storageMock.unlinkDir.mockRejectedValue(new Error('Read only filesystem'));
+
+      await sut.handleUserDelete({ user });
+
+      expect(userRepositoryMock.delete).not.toHaveBeenCalled();
+    });
+  });
 });

server/libs/domain/src/user/user.service.ts

@@ -1,26 +1,43 @@
-import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
+import { UserEntity } from '@app/infra/db/entities';
+import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
 import { randomBytes } from 'crypto';
 import { ReadStream } from 'fs';
+import { join } from 'path';
+import { APP_UPLOAD_LOCATION } from '@app/common';
+import { IAlbumRepository } from '../album/album.repository';
+import { IKeyRepository } from '../api-key/api-key.repository';
+import { IAssetRepository } from '../asset/asset.repository';
 import { AuthUserDto } from '../auth';
-import { ICryptoRepository } from '../crypto';
-import { IUserRepository } from '../user';
-import { CreateUserDto } from './dto/create-user.dto';
-import { UpdateUserDto } from './dto/update-user.dto';
-import { UserCountDto } from './dto/user-count.dto';
+import { ICryptoRepository } from '../crypto/crypto.repository';
+import { IJobRepository, IUserDeletionJob, JobName } from '../job';
+import { IStorageRepository } from '../storage/storage.repository';
+import { IUserTokenRepository } from '../user-token/user-token.repository';
+import { IUserRepository } from '../user/user.repository';
+import { CreateUserDto, UpdateUserDto, UserCountDto } from './dto';
 import {
   CreateProfileImageResponseDto,
   mapCreateProfileImageResponse,
-} from './response-dto/create-profile-image-response.dto';
-import { mapUserCountResponse, UserCountResponseDto } from './response-dto/user-count-response.dto';
-import { mapUser, UserResponseDto } from './response-dto/user-response.dto';
+  mapUser,
+  mapUserCountResponse,
+  UserCountResponseDto,
+  UserResponseDto,
+} from './response-dto';
 import { UserCore } from './user.core';
 
 @Injectable()
 export class UserService {
+  private logger = new Logger(UserService.name);
   private userCore: UserCore;
   constructor(
-    @Inject(IUserRepository) userRepository: IUserRepository,
+    @Inject(IUserRepository) private userRepository: IUserRepository,
     @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
+
+    @Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(IJobRepository) private jobRepository: IJobRepository,
+    @Inject(IKeyRepository) private keyRepository: IKeyRepository,
+    @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+    @Inject(IUserTokenRepository) private tokenRepository: IUserTokenRepository,
   ) {
     this.userCore = new UserCore(userRepository, cryptoRepository);
   }
@@ -123,4 +140,53 @@ export class UserService {
 
     return { admin, password, provided: !!providedPassword };
   }
+
+  async handleUserDeleteCheck() {
+    const users = await this.userRepository.getDeletedUsers();
+    for (const user of users) {
+      if (this.isReadyForDeletion(user)) {
+        await this.jobRepository.queue({ name: JobName.USER_DELETION, data: { user } });
+      }
+    }
+  }
+
+  async handleUserDelete(data: IUserDeletionJob) {
+    const { user } = data;
+
+    // just for extra protection here
+    if (!this.isReadyForDeletion(user)) {
+      this.logger.warn(`Skipped user that was not ready for deletion: id=${user.id}`);
+      return;
+    }
+
+    this.logger.log(`Deleting user: ${user.id}`);
+
+    try {
+      const userAssetDir = join(APP_UPLOAD_LOCATION, user.id);
+      this.logger.warn(`Removing user from filesystem: ${userAssetDir}`);
+      await this.storageRepository.unlinkDir(userAssetDir, { recursive: true, force: true });
+
+      this.logger.warn(`Removing user from database: ${user.id}`);
+
+      await this.tokenRepository.deleteAll(user.id);
+      await this.keyRepository.deleteAll(user.id);
+      await this.albumRepository.deleteAll(user.id);
+      await this.assetRepository.deleteAll(user.id);
+      await this.userRepository.delete(user, true);
+    } catch (error: any) {
+      this.logger.error(`Failed to remove user`, error, { id: user.id });
+    }
+  }
+
+  private isReadyForDeletion(user: UserEntity): boolean {
+    if (!user.deletedAt) {
+      return false;
+    }
+
+    const msInDay = 86400000;
+    const msDeleteWait = msInDay * 7;
+    const msSinceDelete = new Date().getTime() - (Date.parse(user.deletedAt.toString()) || 0);
+
+    return msSinceDelete >= msDeleteWait;
+  }
 }