refactor(server): reverse geocoding (#2167)

* refactor(server): reverse geocoding

* fix: nullable results

server/libs/domain/src/index.ts

@@ -11,6 +11,7 @@ export * from './domain.module';
 export * from './domain.util';
 export * from './job';
 export * from './media';
+export * from './metadata';
 export * from './oauth';
 export * from './search';
 export * from './server-info';

server/libs/domain/src/job/job.constants.ts

@@ -33,7 +33,6 @@ export enum JobName {
   QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction',
   EXIF_EXTRACTION = 'exif-extraction',
   EXTRACT_VIDEO_METADATA = 'extract-video-metadata',
-  REVERSE_GEOCODING = 'reverse-geocoding',
 
   // user deletion
   USER_DELETION = 'user-deletion',

server/libs/domain/src/job/job.interface.ts

@@ -28,11 +28,3 @@ export interface IDeleteFilesJob extends IBaseJob {
 export interface IUserDeletionJob extends IBaseJob {
   user: UserEntity;
 }
-
-export interface IReverseGeocodingJob extends IBaseJob {
-  assetId: string;
-  latitude: number;
-  longitude: number;
-}
-
-export type IMetadataExtractionJob = IAssetUploadedJob | IReverseGeocodingJob;

server/libs/domain/src/job/job.repository.ts

@@ -5,7 +5,6 @@ import {
   IBaseJob,
   IBulkEntityJob,
   IDeleteFilesJob,
-  IReverseGeocodingJob,
   IUserDeletionJob,
 } from './job.interface';
 
@@ -49,7 +48,6 @@ export type JobItem =
   | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
   | { name: JobName.EXIF_EXTRACTION; data: IAssetUploadedJob }
   | { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetUploadedJob }
-  | { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingJob }
 
   // Object Tagging
   | { name: JobName.QUEUE_OBJECT_TAGGING; data: IBaseJob }

server/libs/domain/src/metadata/geocoding.repository.ts

@@ -0,0 +1,17 @@
+export const IGeocodingRepository = 'IGeocodingRepository';
+
+export interface GeoPoint {
+  latitude: number;
+  longitude: number;
+}
+
+export interface ReverseGeocodeResult {
+  country: string | null;
+  state: string | null;
+  city: string | null;
+}
+
+export interface IGeocodingRepository {
+  init(): Promise<void>;
+  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
+}

server/libs/domain/src/metadata/index.ts

@@ -0,0 +1 @@
+export * from './geocoding.repository';

server/libs/infra/src/infra.config.ts

@@ -1,6 +1,7 @@
 import { QueueName } from '@app/domain';
 import { BullModuleOptions } from '@nestjs/bull';
 import { RedisOptions } from 'ioredis';
+import { InitOptions } from 'local-reverse-geocoder';
 import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
 
 function parseRedisConfig(): RedisOptions {
@@ -69,3 +70,21 @@ function parseTypeSenseConfig(): ConfigurationOptions {
 }
 
 export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig();
+
+function parseLocalGeocodingConfig(): InitOptions {
+  const precision = Number(process.env.REVERSE_GEOCODING_PRECISION);
+
+  return {
+    citiesFileOverride: precision ? ['cities15000', 'cities5000', 'cities1000', 'cities500'][precision] : undefined,
+    load: {
+      admin1: true,
+      admin2: true,
+      admin3And4: false,
+      alternateNames: false,
+    },
+    countries: [],
+    dumpDirectory: process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/',
+  };
+}
+
+export const localGeocodingConfig: InitOptions = parseLocalGeocodingConfig();

server/libs/infra/src/infra.module.ts

@@ -4,6 +4,7 @@ import {
   ICommunicationRepository,
   ICryptoRepository,
   IDeviceInfoRepository,
+  IGeocodingRepository,
   IJobRepository,
   IKeyRepository,
   IMachineLearningRepository,
@@ -33,6 +34,7 @@ import {
   CryptoRepository,
   DeviceInfoRepository,
   FilesystemProvider,
+  GeocodingRepository,
   JobRepository,
   MachineLearningRepository,
   MediaRepository,
@@ -50,8 +52,9 @@ const providers: Provider[] = [
   { provide: ICommunicationRepository, useClass: CommunicationRepository },
   { provide: ICryptoRepository, useClass: CryptoRepository },
   { provide: IDeviceInfoRepository, useClass: DeviceInfoRepository },
-  { provide: IKeyRepository, useClass: APIKeyRepository },
+  { provide: IGeocodingRepository, useClass: GeocodingRepository },
   { provide: IJobRepository, useClass: JobRepository },
+  { provide: IKeyRepository, useClass: APIKeyRepository },
   { provide: IMachineLearningRepository, useClass: MachineLearningRepository },
   { provide: IMediaRepository, useClass: MediaRepository },
   { provide: ISearchRepository, useClass: TypesenseRepository },

server/libs/infra/src/repositories/geocoding.repository.ts

@@ -0,0 +1,44 @@
+import { GeoPoint, ReverseGeocodeResult } from '@app/domain';
+import { localGeocodingConfig } from '@app/infra';
+import { Injectable, Logger } from '@nestjs/common';
+import { getName } from 'i18n-iso-countries';
+import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder';
+import { promisify } from 'util';
+
+export interface AdminCode {
+  name: string;
+  asciiName: string;
+  geoNameId: string;
+}
+
+export type GeoData = AddressObject & {
+  admin1Code?: AdminCode | string;
+  admin2Code?: AdminCode | string;
+};
+
+const init = (options: InitOptions): Promise<void> => new Promise<void>((resolve) => geocoder.init(options, resolve));
+const lookup = promisify<GeoPoint[], number, AddressObject[][]>(geocoder.lookUp).bind(geocoder);
+
+@Injectable()
+export class GeocodingRepository {
+  private logger = new Logger(GeocodingRepository.name);
+
+  async init(): Promise<void> {
+    await init(localGeocodingConfig);
+  }
+
+  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
+    this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
+
+    const [address] = await lookup([point], 1);
+    this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`);
+
+    const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData;
+    const country = getName(countryCode, 'en');
+    const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name);
+    const state = stateParts.length > 0 ? stateParts.join(', ') : null;
+    this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`);
+
+    return { country, state, city };
+  }
+}

server/libs/infra/src/repositories/index.ts

@@ -5,6 +5,7 @@ export * from './communication.repository';
 export * from './crypto.repository';
 export * from './device-info.repository';
 export * from './filesystem.provider';
+export * from './geocoding.repository';
 export * from './job.repository';
 export * from './machine-learning.repository';
 export * from './media.repository';

server/libs/infra/src/repositories/job.repository.ts

@@ -1,8 +1,8 @@
 import {
   IAssetJob,
+  IAssetUploadedJob,
   IBaseJob,
   IJobRepository,
-  IMetadataExtractionJob,
   JobCounts,
   JobItem,
   JobName,
@@ -30,7 +30,7 @@ export class JobRepository implements IJobRepository {
     @InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue,
     @InjectQueue(QueueName.OBJECT_TAGGING) private objectTagging: Queue<IAssetJob | IBaseJob>,
     @InjectQueue(QueueName.CLIP_ENCODING) private clipEmbedding: Queue<IAssetJob | IBaseJob>,
-    @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IMetadataExtractionJob | IBaseJob>,
+    @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IAssetUploadedJob | IBaseJob>,
     @InjectQueue(QueueName.STORAGE_TEMPLATE_MIGRATION) private storageTemplateMigration: Queue,
     @InjectQueue(QueueName.THUMBNAIL_GENERATION) private generateThumbnail: Queue,
     @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IAssetJob | IBaseJob>,
@@ -88,7 +88,6 @@ export class JobRepository implements IJobRepository {
       case JobName.QUEUE_METADATA_EXTRACTION:
       case JobName.EXIF_EXTRACTION:
       case JobName.EXTRACT_VIDEO_METADATA:
-      case JobName.REVERSE_GEOCODING:
         await this.metadataExtraction.add(item.name, item.data);
         break;