test(server): full backend end-to-end testing with microservices (#4225)

* feat: asset e2e with job option

* feat: checkout test assets

* feat: library e2e tests

* fix: use node 21 in e2e

* fix: tests

* fix: use normalized external path

* feat: more external path tests

* chore: use parametrized tests

* chore: remove unused test code

* chore: refactor test asset path

* feat: centralize test app creation

* fix: correct error message for missing assets

* feat: test file formats

* fix: don't compare checksum

* feat: build libvips

* fix: install meson

* fix: use immich test asset repo

* feat: test nikon raw files

* fix: set Z timezone

* feat: test offline library files

* feat: richer metadata tests

* feat: e2e tests in docker

* feat: e2e test with arm64 docker

* fix: manual docker compose run

* fix: remove metadata processor import

* fix: run e2e tests in test.yml

* fix: checkout e2e assets

* fix: typo

* fix: checkout files in app directory

* fix: increase e2e memory

* fix: rm submodules

* fix: revert action name

* test: mark file offline when external path changes

* feat: rename env var to TEST_ENV

* docs: new test procedures

* feat: can run docker e2e tests manually if needed

* chore: use new node 20.8 for e2e

* chore: bump exiftool-vendored

* feat: simplify test launching

* fix: rename env vars to use immich_ prefix

* feat: asset folder is submodule

* chore: cleanup after 20.8 upgrade

* fix: don't log postgres in e2e

* fix: better warning about not running all tests

---------

Co-authored-by: Jonathan Jogenfors <jonathan@jogenfors.se>

server/src/domain/job/job.repository.ts

@@ -107,11 +107,12 @@ export type JobItem =
   | { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob };
 
 export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
+export type JobItemHandler = (item: JobItem) => Promise<void>;
 
 export const IJobRepository = 'IJobRepository';
 
 export interface IJobRepository {
-  addHandler(queueName: QueueName, concurrency: number, handler: (job: JobItem) => Promise<void>): void;
+  addHandler(queueName: QueueName, concurrency: number, handler: JobItemHandler): void;
   setConcurrency(queueName: QueueName, concurrency: number): void;
   queue(item: JobItem): Promise<void>;
   pause(name: QueueName): Promise<void>;

server/src/domain/library/library.service.spec.ts

@@ -1172,7 +1172,7 @@ describe(LibraryService.name, () => {
     });
   });
 
-  describe('handleEmptyTrash', () => {
+  describe('handleRemoveOfflineFiles', () => {
     it('can queue trash deletion jobs', async () => {
       assetMock.getWith.mockResolvedValue({ items: [assetStub.image1], hasNextPage: false });
       assetMock.getById.mockResolvedValue(assetStub.image1);

server/src/domain/library/library.service.ts

@@ -363,6 +363,8 @@ export class LibraryService {
       return false;
     }
 
+    const normalizedExternalPath = path.normalize(user.externalPath);
+
     this.logger.verbose(`Refreshing library: ${job.id}`);
     const crawledAssetPaths = (
       await this.storageRepository.crawl({
@@ -373,7 +375,7 @@ export class LibraryService {
       .map(path.normalize)
       .filter((assetPath) =>
         // Filter out paths that are not within the user's external path
-        assetPath.match(new RegExp(`^${user.externalPath}`)),
+        assetPath.match(new RegExp(`^${normalizedExternalPath}`)),
       );
 
     this.logger.debug(`Found ${crawledAssetPaths.length} assets when crawling import paths ${library.importPaths}`);

server/src/immich/api-v1/asset/asset.service.ts

@@ -119,7 +119,7 @@ export class AssetService {
       }
 
       this.logger.error(`Error uploading file ${error}`, error?.stack);
-      throw new BadRequestException(`Error uploading file`, `${error}`);
+      throw error;
     }
   }
 

server/src/infra/infra.config.ts

@@ -5,6 +5,10 @@ import { RedisOptions } from 'ioredis';
 import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
 
 function parseRedisConfig(): RedisOptions {
+  if (process.env.IMMICH_TEST_ENV == 'true') {
+    return {};
+  }
+
   const redisUrl = process.env.REDIS_URL;
   if (redisUrl && redisUrl.startsWith('ioredis://')) {
     try {

server/src/infra/infra.module.ts

@@ -80,16 +80,24 @@ const providers: Provider[] = [
   { provide: IUserTokenRepository, useClass: UserTokenRepository },
 ];
 
+const imports = [
+  ConfigModule.forRoot(immichAppConfig),
+  TypeOrmModule.forRoot(databaseConfig),
+  TypeOrmModule.forFeature(databaseEntities),
+];
+
+const moduleExports = [...providers];
+
+if (process.env.IMMICH_TEST_ENV !== 'true') {
+  imports.push(BullModule.forRoot(bullConfig));
+  imports.push(BullModule.registerQueue(...bullQueues));
+  moduleExports.push(BullModule);
+}
+
 @Global()
 @Module({
-  imports: [
-    ConfigModule.forRoot(immichAppConfig),
-    TypeOrmModule.forRoot(databaseConfig),
-    TypeOrmModule.forFeature(databaseEntities),
-    BullModule.forRoot(bullConfig),
-    BullModule.registerQueue(...bullQueues),
-  ],
+  imports,
   providers: [...providers],
-  exports: [...providers, BullModule],
+  exports: moduleExports,
 })
 export class InfraModule {}