feat(cli) Add new CLI (#3066)

* Add new cli

* Remove old readme

* Add documentation to readme file

* Add github workflow tests for cli

* Fix typo in docs

* Add usage info to readme

* Add package-lock.json

* Fix tsconfig

* Cleanup

* Fix lint

* Cleanup package.json

* Fix accidental server change

* Remove rootdir from cli

* Remove tsbuildinfo

* Add prettierignore

* Make CLI use internal openapi specs

* Add ignore and dry-run features

* Sort paths alphabetically

* Don't remove substring

* Remove shorthand for delete

* Remove unused import

* Remove chokidar

* Set correct openapi cli generator script

* Add progress bar

* Rename target to asset

* Add deletion progress bar

* Ignore require statement

* Use read streams instead of readfile

* Fix github feedback

* Fix upload requires

* More github comments

* Cleanup messages

* Cleaner pattern concats

* Github comments

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

cli/src/cores/api-configuration.ts

@@ -0,0 +1,9 @@
+export class ApiConfiguration {
+  public readonly instanceUrl!: string;
+  public readonly apiKey!: string;
+
+  constructor(instanceUrl: string, apiKey: string) {
+    this.instanceUrl = instanceUrl;
+    this.apiKey = apiKey;
+  }
+}

cli/src/cores/constants.ts

@@ -0,0 +1,58 @@
+// Check asset-upload.config.spec.ts for complete list
+// TODO: we should get this list from the server via API in the future
+
+// Videos
+const videos = ['mp4', 'webm', 'mov', '3gp', 'avi', 'm2ts', 'mts', 'mpg', 'flv', 'mkv', 'wmv'];
+
+// Images
+const heic = ['heic', 'heif'];
+const jpeg = ['jpg', 'jpeg'];
+const png = ['png'];
+const gif = ['gif'];
+const tiff = ['tif', 'tiff'];
+const webp = ['webp'];
+const dng = ['dng'];
+const other = [
+  '3fr',
+  'ari',
+  'arw',
+  'avif',
+  'cap',
+  'cin',
+  'cr2',
+  'cr3',
+  'crw',
+  'dcr',
+  'nef',
+  'erf',
+  'fff',
+  'iiq',
+  'jxl',
+  'k25',
+  'kdc',
+  'mrw',
+  'orf',
+  'ori',
+  'pef',
+  'raf',
+  'raw',
+  'rwl',
+  'sr2',
+  'srf',
+  'srw',
+  'orf',
+  'ori',
+  'x3f',
+];
+
+export const ACCEPTED_FILE_EXTENSIONS = [
+  ...videos,
+  ...jpeg,
+  ...png,
+  ...heic,
+  ...gif,
+  ...tiff,
+  ...webp,
+  ...dng,
+  ...other,
+];

cli/src/cores/dto/crawl-options-dto.ts

@@ -0,0 +1,6 @@
+export class CrawlOptionsDto {
+  pathsToCrawl!: string[];
+  recursive = false;
+  includeHidden = false;
+  excludePatterns!: string[];
+}

cli/src/cores/dto/upload-options-dto.ts

@@ -0,0 +1,8 @@
+export class UploadOptionsDto {
+  recursive = false;
+  excludePatterns!: string[];
+  dryRun = false;
+  skipHash = false;
+  delete = false;
+  import = false;
+}

cli/src/cores/errors/login-error.ts

@@ -0,0 +1,11 @@
+export class LoginError extends Error {
+  constructor(message: string) {
+    super(message);
+
+    // assign the error class name in your custom error (as a shortcut)
+    this.name = this.constructor.name;
+
+    // capturing the stack trace keeps the reference to your error class
+    Error.captureStackTrace(this, this.constructor);
+  }
+}

cli/src/cores/index.ts

@@ -0,0 +1,2 @@
+export * from './constants';
+export * from './models';

cli/src/cores/models/crawled-asset.ts

@@ -0,0 +1,71 @@
+import * as fs from 'fs';
+import * as mime from 'mime-types';
+import { basename } from 'node:path';
+import * as path from 'path';
+import crypto from 'crypto';
+import { AssetTypeEnum } from 'src/api/open-api';
+
+export class CrawledAsset {
+  public path: string;
+
+  public assetType?: AssetTypeEnum;
+  public assetData?: fs.ReadStream;
+  public deviceAssetId?: string;
+  public fileCreatedAt?: string;
+  public fileModifiedAt?: string;
+  public fileExtension?: string;
+  public sidecarData?: Buffer;
+  public sidecarPath?: string;
+  public fileSize!: number;
+  public skipped = false;
+
+  constructor(path: string) {
+    this.path = path;
+  }
+
+  async readData() {
+    this.assetData = fs.createReadStream(this.path);
+  }
+
+  async process() {
+    const stats = await fs.promises.stat(this.path);
+    this.deviceAssetId = `${basename(this.path)}-${stats.size}`.replace(/\s+/g, '');
+
+    // TODO: Determine file type from extension only
+    const mimeType = mime.lookup(this.path);
+    if (!mimeType) {
+      throw Error('Cannot determine mime type of asset: ' + this.path);
+    }
+    this.assetType = mimeType.split('/')[0].toUpperCase() as AssetTypeEnum;
+    this.fileCreatedAt = stats.ctime.toISOString();
+    this.fileModifiedAt = stats.mtime.toISOString();
+    this.fileExtension = path.extname(this.path);
+    this.fileSize = stats.size;
+
+    // TODO: doesn't xmp replace the file extension? Will need investigation
+    const sideCarPath = `${this.path}.xmp`;
+    try {
+      fs.accessSync(sideCarPath, fs.constants.R_OK);
+      this.sidecarData = await fs.promises.readFile(sideCarPath);
+      this.sidecarPath = sideCarPath;
+    } catch (error) {}
+  }
+
+  async delete(): Promise<void> {
+    return fs.promises.unlink(this.path);
+  }
+
+  public async hash(): Promise<string> {
+    const sha1 = (filePath: string) => {
+      const hash = crypto.createHash('sha1');
+      return new Promise<string>((resolve, reject) => {
+        const rs = fs.createReadStream(filePath);
+        rs.on('error', reject);
+        rs.on('data', (chunk) => hash.update(chunk));
+        rs.on('end', () => resolve(hash.digest('hex')));
+      });
+    };
+
+    return await sha1(this.path);
+  }
+}

cli/src/cores/models/index.ts

@@ -0,0 +1 @@
+export * from './crawled-asset';