feat(cli) Add new CLI (#3066)

* Add new cli

* Remove old readme

* Add documentation to readme file

* Add github workflow tests for cli

* Fix typo in docs

* Add usage info to readme

* Add package-lock.json

* Fix tsconfig

* Cleanup

* Fix lint

* Cleanup package.json

* Fix accidental server change

* Remove rootdir from cli

* Remove tsbuildinfo

* Add prettierignore

* Make CLI use internal openapi specs

* Add ignore and dry-run features

* Sort paths alphabetically

* Don't remove substring

* Remove shorthand for delete

* Remove unused import

* Remove chokidar

* Set correct openapi cli generator script

* Add progress bar

* Rename target to asset

* Add deletion progress bar

* Ignore require statement

* Use read streams instead of readfile

* Fix github feedback

* Fix upload requires

* More github comments

* Cleanup messages

* Cleaner pattern concats

* Github comments

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

cli/src/services/crawl.service.spec.ts

@@ -0,0 +1,235 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { CrawlService } from './crawl.service';
+import mockfs from 'mock-fs';
+import { toIncludeSameMembers } from 'jest-extended';
+import { CrawlOptionsDto } from '../cores/dto/crawl-options-dto';
+
+const matchers = require('jest-extended');
+expect.extend(matchers);
+
+const crawlService = new CrawlService();
+
+describe('CrawlService', () => {
+  beforeAll(() => {
+    // Write a dummy output before mock-fs to prevent some annoying errors
+    console.log();
+  });
+
+  it('should crawl a single directory', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should crawl a single file', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/image.jpg'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should crawl a file and a directory', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/images/photo.jpg': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/image.jpg', '/images/'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg', '/images/photo.jpg']);
+  });
+
+  it('should exclude by file extension', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/image.tif': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    options.excludePatterns = ['**/*.tif'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should exclude by file extension without case sensitivity', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/image.tif': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    options.excludePatterns = ['**/*.TIF'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should exclude by folder', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/raw/image.jpg': '',
+      '/photos/raw2/image.jpg': '',
+      '/photos/folder/raw/image.jpg': '',
+      '/photos/crawl/image.jpg': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    options.excludePatterns = ['**/raw/**'];
+    options.recursive = true;
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg', '/photos/raw2/image.jpg', '/photos/crawl/image.jpg']);
+  });
+
+  it('should crawl multiple paths', async () => {
+    mockfs({
+      '/photos/image1.jpg': '',
+      '/images/image2.jpg': '',
+      '/albums/image3.jpg': '',
+    });
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/', '/images/', '/albums/'];
+    options.recursive = false;
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image1.jpg', '/images/image2.jpg', '/albums/image3.jpg']);
+  });
+
+  it('should crawl a single path without trailing slash', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+    });
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should crawl a single path without recursion', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/subfolder/image1.jpg': '',
+      '/photos/subfolder/image2.jpg': '',
+      '/image1.jpg': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should crawl a single path with recursion', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/subfolder/image1.jpg': '',
+      '/photos/subfolder/image2.jpg': '',
+      '/image1.jpg': '',
+    });
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    options.recursive = true;
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers([
+      '/photos/image.jpg',
+      '/photos/subfolder/image1.jpg',
+      '/photos/subfolder/image2.jpg',
+    ]);
+  });
+
+  it('should filter file extensions', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/image.txt': '',
+      '/photos/1': '',
+    });
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
+  });
+
+  it('should include photo and video extensions', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/image.jpeg': '',
+      '/photos/image.heic': '',
+      '/photos/image.heif': '',
+      '/photos/image.png': '',
+      '/photos/image.gif': '',
+      '/photos/image.tif': '',
+      '/photos/image.tiff': '',
+      '/photos/image.webp': '',
+      '/photos/image.dng': '',
+      '/photos/image.nef': '',
+      '/videos/video.mp4': '',
+      '/videos/video.mov': '',
+      '/videos/video.webm': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/', '/videos/'];
+    const paths: string[] = await crawlService.crawl(options);
+
+    expect(paths).toIncludeSameMembers([
+      '/photos/image.jpg',
+      '/photos/image.jpeg',
+      '/photos/image.heic',
+      '/photos/image.heif',
+      '/photos/image.png',
+      '/photos/image.gif',
+      '/photos/image.tif',
+      '/photos/image.tiff',
+      '/photos/image.webp',
+      '/photos/image.dng',
+      '/photos/image.nef',
+      '/videos/video.mp4',
+      '/videos/video.mov',
+      '/videos/video.webm',
+    ]);
+  });
+
+  it('should check file extensions without case sensitivity', async () => {
+    mockfs({
+      '/photos/image.jpg': '',
+      '/photos/image.Jpg': '',
+      '/photos/image.jpG': '',
+      '/photos/image.JPG': '',
+      '/photos/image.jpEg': '',
+      '/photos/image.TIFF': '',
+      '/photos/image.tif': '',
+      '/photos/image.dng': '',
+      '/photos/image.NEF': '',
+    });
+
+    const options = new CrawlOptionsDto();
+    options.pathsToCrawl = ['/photos/'];
+    const paths: string[] = await crawlService.crawl(options);
+    expect(paths).toIncludeSameMembers([
+      '/photos/image.jpg',
+      '/photos/image.Jpg',
+      '/photos/image.jpG',
+      '/photos/image.JPG',
+      '/photos/image.jpEg',
+      '/photos/image.TIFF',
+      '/photos/image.tif',
+      '/photos/image.dng',
+      '/photos/image.NEF',
+    ]);
+  });
+
+  afterEach(() => {
+    mockfs.restore();
+  });
+});

cli/src/services/crawl.service.ts

@@ -0,0 +1,47 @@
+import { CrawlOptionsDto } from 'src/cores/dto/crawl-options-dto';
+import { ACCEPTED_FILE_EXTENSIONS } from '../cores';
+import { glob } from 'glob';
+import * as fs from 'fs';
+
+export class CrawlService {
+  public async crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
+    const pathsToCrawl: string[] = crawlOptions.pathsToCrawl;
+
+    const directories: string[] = [];
+    const crawledFiles: string[] = [];
+
+    for await (const currentPath of pathsToCrawl) {
+      const stats = await fs.promises.stat(currentPath);
+      if (stats.isFile() || stats.isSymbolicLink()) {
+        crawledFiles.push(currentPath);
+      } else {
+        directories.push(currentPath);
+      }
+    }
+
+    let searchPattern: string;
+    if (directories.length === 1) {
+      searchPattern = directories[0];
+    } else if (directories.length === 0) {
+      return crawledFiles;
+    } else {
+      searchPattern = '{' + directories.join(',') + '}';
+    }
+
+    if (crawlOptions.recursive) {
+      searchPattern = searchPattern + '/**/';
+    }
+
+    searchPattern = `${searchPattern}/*.{${ACCEPTED_FILE_EXTENSIONS.join(',')}}`;
+
+    const globbedFiles = await glob(searchPattern, {
+      nocase: true,
+      nodir: true,
+      ignore: crawlOptions.excludePatterns,
+    });
+
+    const returnedFiles = crawledFiles.concat(globbedFiles);
+    returnedFiles.sort();
+    return returnedFiles;
+  }
+}

cli/src/services/index.ts

@@ -0,0 +1,2 @@
+export * from './upload.service';
+export * from './crawl.service';

cli/src/services/session.service.spec.ts

@@ -0,0 +1,95 @@
+import { SessionService } from './session.service';
+import mockfs from 'mock-fs';
+import fs from 'node:fs';
+import yaml from 'yaml';
+import { LoginError } from '../cores/errors/login-error';
+
+const mockPingServer = jest.fn(() => Promise.resolve({ data: { res: 'pong' } }));
+const mockUserInfo = jest.fn(() => Promise.resolve({ data: { email: 'admin@example.com' } }));
+
+jest.mock('../api/open-api', () => {
+  return {
+    __esModule: true,
+    ...jest.requireActual('../api/open-api'),
+    UserApi: jest.fn().mockImplementation(() => {
+      return { getMyUserInfo: mockUserInfo };
+    }),
+    ServerInfoApi: jest.fn().mockImplementation(() => {
+      return { pingServer: mockPingServer };
+    }),
+  };
+});
+
+describe('SessionService', () => {
+  let sessionService: SessionService;
+  beforeAll(() => {
+    // Write a dummy output before mock-fs to prevent some annoying errors
+    console.log();
+  });
+
+  beforeEach(() => {
+    const configDir = '/config';
+    sessionService = new SessionService(configDir);
+  });
+
+  it('should connect to immich', async () => {
+    mockfs({
+      '/config/auth.yml': 'apiKey: pNussssKSYo5WasdgalvKJ1n9kdvaasdfbluPg\ninstanceUrl: https://test/api',
+    });
+    await sessionService.connect();
+    expect(mockPingServer).toHaveBeenCalledTimes(1);
+  });
+
+  it('should error if no auth file exists', async () => {
+    mockfs();
+    await sessionService.connect().catch((error) => {
+      expect(error.message).toEqual('No auth file exist. Please login first');
+    });
+  });
+
+  it('should error if auth file is missing instance URl', async () => {
+    mockfs({
+      '/config/auth.yml': 'foo: pNussssKSYo5WasdgalvKJ1n9kdvaasdfbluPg\napiKey: https://test/api',
+    });
+    await sessionService.connect().catch((error) => {
+      expect(error).toBeInstanceOf(LoginError);
+      expect(error.message).toEqual('Instance URL missing in auth config file /config/auth.yml');
+    });
+  });
+
+  it('should error if auth file is missing api key', async () => {
+    mockfs({
+      '/config/auth.yml': 'instanceUrl: pNussssKSYo5WasdgalvKJ1n9kdvaasdfbluPg\nbar: https://test/api',
+    });
+    await sessionService.connect().catch((error) => {
+      expect(error).toBeInstanceOf(LoginError);
+      expect(error.message).toEqual('API key missing in auth config file /config/auth.yml');
+    });
+  });
+
+  it('should create auth file when logged in', async () => {
+    mockfs();
+
+    await sessionService.keyLogin('https://test/api', 'pNussssKSYo5WasdgalvKJ1n9kdvaasdfbluPg');
+
+    const data: string = await fs.promises.readFile('/config/auth.yml', 'utf8');
+    const authConfig = yaml.parse(data);
+    expect(authConfig.instanceUrl).toBe('https://test/api');
+    expect(authConfig.apiKey).toBe('pNussssKSYo5WasdgalvKJ1n9kdvaasdfbluPg');
+  });
+
+  it('should delete auth file when logging out', async () => {
+    mockfs({
+      '/config/auth.yml': 'apiKey: pNussssKSYo5WasdgalvKJ1n9kdvaasdfbluPg\ninstanceUrl: https://test/api',
+    });
+    await sessionService.logout();
+
+    await fs.promises.access('/auth.yml', fs.constants.F_OK).catch((error) => {
+      expect(error.message).toContain('ENOENT');
+    });
+  });
+
+  afterEach(() => {
+    mockfs.restore();
+  });
+});

cli/src/services/session.service.ts

@@ -0,0 +1,81 @@
+import fs from 'node:fs';
+import yaml from 'yaml';
+import path from 'node:path';
+import { ImmichApi } from '../api/client';
+import { LoginError } from '../cores/errors/login-error';
+
+export class SessionService {
+  readonly configDir: string;
+  readonly authPath!: string;
+  private api!: ImmichApi;
+
+  constructor(configDir: string) {
+    this.configDir = configDir;
+    this.authPath = path.join(this.configDir, 'auth.yml');
+  }
+
+  public async connect(): Promise<ImmichApi> {
+    await fs.promises.access(this.authPath, fs.constants.F_OK).catch((error) => {
+      if (error.code === 'ENOENT') {
+        throw new LoginError('No auth file exist. Please login first');
+      }
+    });
+
+    const data: string = await fs.promises.readFile(this.authPath, 'utf8');
+    const parsedConfig = yaml.parse(data);
+    const instanceUrl: string = parsedConfig.instanceUrl;
+    const apiKey: string = parsedConfig.apiKey;
+
+    if (!instanceUrl) {
+      throw new LoginError('Instance URL missing in auth config file ' + this.authPath);
+    }
+
+    if (!apiKey) {
+      throw new LoginError('API key missing in auth config file ' + this.authPath);
+    }
+
+    this.api = new ImmichApi(instanceUrl, apiKey);
+
+    await this.ping();
+
+    return this.api;
+  }
+
+  public async keyLogin(instanceUrl: string, apiKey: string): Promise<ImmichApi> {
+    this.api = new ImmichApi(instanceUrl, apiKey);
+
+    // Check if server and api key are valid
+    const { data: userInfo } = await this.api.userApi.getMyUserInfo().catch((error) => {
+      throw new LoginError(`Failed to connect to the server: ${error.message}`);
+    });
+
+    console.log(`Logged in as ${userInfo.email}`);
+
+    if (!fs.existsSync(this.configDir)) {
+      // Create config folder if it doesn't exist
+      fs.mkdirSync(this.configDir, { recursive: true });
+    }
+
+    fs.writeFileSync(this.authPath, yaml.stringify({ instanceUrl, apiKey }));
+
+    console.log('Wrote auth info to ' + this.authPath);
+    return this.api;
+  }
+
+  public async logout(): Promise<void> {
+    if (fs.existsSync(this.authPath)) {
+      fs.unlinkSync(this.authPath);
+      console.log('Removed auth file ' + this.authPath);
+    }
+  }
+
+  private async ping(): Promise<void> {
+    const { data: pingResponse } = await this.api.serverInfoApi.pingServer().catch((error) => {
+      throw new Error(`Failed to connect to the server: ${error.message}`);
+    });
+
+    if (pingResponse.res !== 'pong') {
+      throw new Error('Unexpected ping reply');
+    }
+  }
+}

cli/src/services/upload.service.spec.ts

@@ -0,0 +1,36 @@
+import { UploadService } from './upload.service';
+import mockfs from 'mock-fs';
+import axios from 'axios';
+import mockAxios from 'jest-mock-axios';
+import FormData from 'form-data';
+import { ApiConfiguration } from '../cores/api-configuration';
+
+describe('UploadService', () => {
+  let uploadService: UploadService;
+
+  beforeAll(() => {
+    // Write a dummy output before mock-fs to prevent some annoying errors
+    console.log();
+  });
+
+  beforeEach(() => {
+    const apiConfiguration = new ApiConfiguration('https://example.com/api', 'key');
+
+    uploadService = new UploadService(apiConfiguration);
+  });
+
+  it('should upload a single file', async () => {
+    const data = new FormData();
+    data.append('assetType', 'image');
+
+    uploadService.upload(data);
+
+    mockAxios.mockResponse();
+    expect(axios).toHaveBeenCalled();
+  });
+
+  afterEach(() => {
+    mockfs.restore();
+    mockAxios.reset();
+  });
+});

cli/src/services/upload.service.ts

@@ -0,0 +1,65 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import FormData from 'form-data';
+import { ApiConfiguration } from '../cores/api-configuration';
+
+export class UploadService {
+  private readonly uploadConfig: AxiosRequestConfig<any>;
+  private readonly checkAssetExistenceConfig: AxiosRequestConfig<any>;
+  private readonly importConfig: AxiosRequestConfig<any>;
+
+  constructor(apiConfiguration: ApiConfiguration) {
+    this.uploadConfig = {
+      method: 'post',
+      maxRedirects: 0,
+      url: `${apiConfiguration.instanceUrl}/asset/upload`,
+      headers: {
+        'x-api-key': apiConfiguration.apiKey,
+      },
+      maxContentLength: Number.POSITIVE_INFINITY,
+      maxBodyLength: Number.POSITIVE_INFINITY,
+    };
+
+    this.importConfig = {
+      method: 'post',
+      maxRedirects: 0,
+      url: `${apiConfiguration.instanceUrl}/asset/import`,
+      headers: {
+        'x-api-key': apiConfiguration.apiKey,
+        'Content-Type': 'application/json',
+      },
+      maxContentLength: Number.POSITIVE_INFINITY,
+      maxBodyLength: Number.POSITIVE_INFINITY,
+    };
+
+    this.checkAssetExistenceConfig = {
+      method: 'post',
+      maxRedirects: 0,
+      url: `${apiConfiguration.instanceUrl}/asset/bulk-upload-check`,
+      headers: {
+        'x-api-key': apiConfiguration.apiKey,
+        'Content-Type': 'application/json',
+      },
+    };
+  }
+
+  public checkIfAssetAlreadyExists(path: string, checksum: string): Promise<any> {
+    this.checkAssetExistenceConfig.data = JSON.stringify({ assets: [{ id: path, checksum: checksum }] });
+
+    // TODO: retry on 500 errors?
+    return axios(this.checkAssetExistenceConfig);
+  }
+
+  public upload(data: FormData): Promise<any> {
+    this.uploadConfig.data = data;
+
+    // TODO: retry on 500 errors?
+    return axios(this.uploadConfig);
+  }
+
+  public import(data: any): Promise<any> {
+    this.importConfig.data = data;
+
+    // TODO: retry on 500 errors?
+    return axios(this.importConfig);
+  }
+}