feature(mobile): sync assets, albums & users to local database on device (#1759)

* feature(mobile): sync assets, albums & users to local database on device

* try to fix tests

* move DB sync operations to new SyncService

* clear db on user logout

* fix reason for endless loading timeline

* fix error when deleting album

* fix thumbnail of device albums

* add a few comments

* fix Hive box not open in album service when loading local assets

* adjust tests to int IDs

* fix bug: show all albums when Recent is selected

* update generated api

* reworked Recents album isAll handling

* guard against wrongly interleaved sync operations

* fix: timeline asset ordering (sort asset state by created at)

* fix: sort assets in albums by created at

mobile/lib/shared/services/api.service.dart

@@ -28,9 +28,13 @@ class ApiService {
       debugPrint("Cannot init ApiServer endpoint, userInfoBox not open yet.");
     }
   }
+  String? _authToken;
 
   setEndpoint(String endpoint) {
     _apiClient = ApiClient(basePath: endpoint);
+    if (_authToken != null) {
+      setAccessToken(_authToken!);
+    }
     userApi = UserApi(_apiClient);
     authenticationApi = AuthenticationApi(_apiClient);
     oAuthApi = OAuthApi(_apiClient);
@@ -94,6 +98,9 @@ class ApiService {
   }
 
   setAccessToken(String accessToken) {
+    _authToken = accessToken;
     _apiClient.addDefaultHeader('Authorization', 'Bearer $accessToken');
   }
+
+  ApiClient get apiClient => _apiClient;
 }

mobile/lib/shared/services/asset.service.dart

@@ -1,101 +1,84 @@
 import 'dart:async';
 
 import 'package:flutter/material.dart';
-import 'package:hive/hive.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
-import 'package:immich_mobile/constants/hive_box.dart';
-import 'package:immich_mobile/modules/backup/background_service/background.service.dart';
-import 'package:immich_mobile/modules/backup/models/hive_backup_albums.model.dart';
-import 'package:immich_mobile/modules/backup/services/backup.service.dart';
 import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/models/exif_info.dart';
 import 'package:immich_mobile/shared/models/store.dart';
+import 'package:immich_mobile/shared/models/user.dart';
 import 'package:immich_mobile/shared/providers/api.provider.dart';
+import 'package:immich_mobile/shared/providers/db.provider.dart';
 import 'package:immich_mobile/shared/services/api.service.dart';
+import 'package:immich_mobile/shared/services/sync.service.dart';
 import 'package:immich_mobile/utils/openapi_extensions.dart';
 import 'package:immich_mobile/utils/tuple.dart';
+import 'package:isar/isar.dart';
 import 'package:logging/logging.dart';
 import 'package:openapi/api.dart';
 
 final assetServiceProvider = Provider(
   (ref) => AssetService(
     ref.watch(apiServiceProvider),
-    ref.watch(backupServiceProvider),
-    ref.watch(backgroundServiceProvider),
+    ref.watch(syncServiceProvider),
+    ref.watch(dbProvider),
   ),
 );
 
 class AssetService {
   final ApiService _apiService;
-  final BackupService _backupService;
-  final BackgroundService _backgroundService;
+  final SyncService _syncService;
   final log = Logger('AssetService');
+  final Isar _db;
 
-  AssetService(this._apiService, this._backupService, this._backgroundService);
+  AssetService(
+    this._apiService,
+    this._syncService,
+    this._db,
+  );
+
+  /// Checks the server for updated assets and updates the local database if
+  /// required. Returns `true` if there were any changes.
+  Future<bool> refreshRemoteAssets() async {
+    final Stopwatch sw = Stopwatch()..start();
+    final int numOwnedRemoteAssets = await _db.assets
+        .where()
+        .remoteIdIsNotNull()
+        .filter()
+        .ownerIdEqualTo(Store.get<User>(StoreKey.currentUser)!.isarId)
+        .count();
+    final List<AssetResponseDto>? dtos =
+        await _getRemoteAssets(hasCache: numOwnedRemoteAssets > 0);
+    if (dtos == null) {
+      debugPrint("refreshRemoteAssets fast took ${sw.elapsedMilliseconds}ms");
+      return false;
+    }
+    final bool changes = await _syncService
+        .syncRemoteAssetsToDb(dtos.map(Asset.remote).toList());
+    debugPrint("refreshRemoteAssets full took ${sw.elapsedMilliseconds}ms");
+    return changes;
+  }
 
   /// Returns `null` if the server state did not change, else list of assets
-  Future<Pair<List<Asset>?, String?>> getRemoteAssets({String? etag}) async {
+  Future<List<AssetResponseDto>?> _getRemoteAssets({
+    required bool hasCache,
+  }) async {
     try {
-      // temporary fix for race condition that the _apiService
-      // get called before accessToken is set
-      var userInfoHiveBox = await Hive.openBox(userInfoBox);
-      var accessToken = userInfoHiveBox.get(accessTokenKey);
-      _apiService.setAccessToken(accessToken);
-
+      final etag = hasCache ? Store.get(StoreKey.assetETag) : null;
       final Pair<List<AssetResponseDto>, String?>? remote =
           await _apiService.assetApi.getAllAssetsWithETag(eTag: etag);
       if (remote == null) {
-        return Pair(null, etag);
+        return null;
       }
-      return Pair(
-        remote.first.map(Asset.remote).toList(growable: false),
-        remote.second,
-      );
+      if (remote.second != null && remote.second != etag) {
+        Store.put(StoreKey.assetETag, remote.second);
+      }
+      return remote.first;
     } catch (e, stack) {
       log.severe('Error while getting remote assets', e, stack);
-      debugPrint("[ERROR] [getRemoteAssets] $e");
-      return Pair(null, etag);
+      return null;
     }
   }
 
-  /// if [urgent] is `true`, do not block by waiting on the background service
-  /// to finish running. Returns `null` instead after a timeout.
-  Future<List<Asset>?> getLocalAssets({bool urgent = false}) async {
-    try {
-      final Future<bool> hasAccess = urgent
-          ? _backgroundService.hasAccess
-              .timeout(const Duration(milliseconds: 250))
-          : _backgroundService.hasAccess;
-      if (!await hasAccess) {
-        throw Exception("Error [getAllAsset] failed to gain access");
-      }
-      final box = await Hive.openBox<HiveBackupAlbums>(hiveBackupInfoBox);
-      final HiveBackupAlbums? backupAlbumInfo = box.get(backupInfoKey);
-      final String userId = Store.get(StoreKey.userRemoteId);
-      if (backupAlbumInfo != null) {
-        return (await _backupService
-                .buildUploadCandidates(backupAlbumInfo.deepCopy()))
-            .map((e) => Asset.local(e, userId))
-            .toList(growable: false);
-      }
-    } catch (e, stackTrace) {
-      log.severe('Error while getting local assets', e, stackTrace);
-      debugPrint("Error [_getLocalAssets] ${e.toString()}");
-    }
-    return null;
-  }
-
-  Future<Asset?> getAssetById(String assetId) async {
-    try {
-      final dto = await _apiService.assetApi.getAssetById(assetId);
-      if (dto != null) {
-        return Asset.remote(dto);
-      }
-    } catch (e) {
-      debugPrint("Error [getAssetById]  ${e.toString()}");
-    }
-    return null;
-  }
-
   Future<List<DeleteAssetResponseDto>?> deleteAssets(
     Iterable<Asset> deleteAssets,
   ) async {
@@ -114,6 +97,28 @@ class AssetService {
     }
   }
 
+  /// Loads the exif information from the database. If there is none, loads
+  /// the exif info from the server (remote assets only)
+  Future<Asset> loadExif(Asset a) async {
+    a.exifInfo ??= await _db.exifInfos.get(a.id);
+    if (a.exifInfo?.iso == null) {
+      if (a.isRemote) {
+        final dto = await _apiService.assetApi.getAssetById(a.remoteId!);
+        if (dto != null && dto.exifInfo != null) {
+          a = a.withUpdatesFromDto(dto);
+          if (a.isInDb) {
+            _db.writeTxn(() => a.put(_db));
+          } else {
+            debugPrint("[loadExif] parameter Asset is not from DB!");
+          }
+        }
+      } else {
+        // TODO implement local exif info parsing
+      }
+    }
+    return a;
+  }
+
   Future<Asset?> updateAsset(
     Asset asset,
     UpdateAssetDto updateAssetDto,

mobile/lib/shared/services/asset_cache.service.dart

@@ -1,41 +1,13 @@
-import 'package:collection/collection.dart';
-import 'package:flutter/foundation.dart';
-import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/shared/models/asset.dart';
 import 'package:immich_mobile/shared/services/json_cache.dart';
 
+@Deprecated("only kept to remove its files after migration")
 class AssetCacheService extends JsonCache<List<Asset>> {
   AssetCacheService() : super("asset_cache");
 
-  static Future<List<Map<String, dynamic>>> _computeSerialize(
-    List<Asset> assets,
-  ) async {
-    return assets.map((e) => e.toJson()).toList();
-  }
+  @override
+  void put(List<Asset> data) {}
 
   @override
-  void put(List<Asset> data) async {
-    putRawData(await compute(_computeSerialize, data));
-  }
-
-  static Future<List<Asset>> _computeEncode(List<dynamic> data) async {
-    return data.map((e) => Asset.fromJson(e)).whereNotNull().toList();
-  }
-
-  @override
-  Future<List<Asset>?> get() async {
-    try {
-      final mapList = await readRawData() as List<dynamic>;
-      final responseData = await compute(_computeEncode, mapList);
-      return responseData;
-    } catch (e) {
-      debugPrint(e.toString());
-      await invalidate();
-      return null;
-    }
-  }
+  Future<List<Asset>?> get() => Future.value(null);
 }
-
-final assetCacheServiceProvider = Provider(
-  (ref) => AssetCacheService(),
-);

mobile/lib/shared/services/json_cache.dart

@@ -1,9 +1,8 @@
-import 'dart:convert';
 import 'dart:io';
 
-import 'package:flutter/foundation.dart';
 import 'package:path_provider/path_provider.dart';
 
+@Deprecated("only kept to remove its files after migration")
 abstract class JsonCache<T> {
   final String cacheFileName;
 
@@ -32,33 +31,6 @@ abstract class JsonCache<T> {
     }
   }
 
-  static Future<String> _computeEncodeJson(dynamic toEncode) async {
-    return json.encode(toEncode);
-  }
-
-  Future<void> putRawData(dynamic data) async {
-    final jsonString = await compute(_computeEncodeJson, data);
-
-    final file = await _getCacheFile();
-
-    if (!await file.exists()) {
-      await file.create();
-    }
-
-    await file.writeAsString(jsonString);
-  }
-
-  static Future<dynamic> _computeDecodeJson(String jsonString) async {
-    return json.decode(jsonString);
-  }
-
-  Future<dynamic> readRawData() async {
-    final file = await _getCacheFile();
-    final data = await file.readAsString();
-
-    return await compute(_computeDecodeJson, data);
-  }
-
   void put(T data);
   Future<T?> get();
 }

mobile/lib/shared/services/sync.service.dart

@@ -0,0 +1,558 @@
+import 'dart:async';
+
+import 'package:collection/collection.dart';
+import 'package:flutter/foundation.dart';
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/shared/models/album.dart';
+import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/models/exif_info.dart';
+import 'package:immich_mobile/shared/models/store.dart';
+import 'package:immich_mobile/shared/models/user.dart';
+import 'package:immich_mobile/shared/providers/db.provider.dart';
+import 'package:immich_mobile/utils/async_mutex.dart';
+import 'package:immich_mobile/utils/diff.dart';
+import 'package:immich_mobile/utils/tuple.dart';
+import 'package:isar/isar.dart';
+import 'package:openapi/api.dart';
+import 'package:photo_manager/photo_manager.dart';
+
+final syncServiceProvider =
+    Provider((ref) => SyncService(ref.watch(dbProvider)));
+
+class SyncService {
+  final Isar _db;
+  final AsyncMutex _lock = AsyncMutex();
+
+  SyncService(this._db);
+
+  // public methods:
+
+  /// Syncs users from the server to the local database
+  /// Returns `true`if there were any changes
+  Future<bool> syncUsersFromServer(List<User> users) async {
+    users.sortBy((u) => u.id);
+    final dbUsers = await _db.users.where().sortById().findAll();
+    final List<int> toDelete = [];
+    final List<User> toUpsert = [];
+    final changes = diffSortedListsSync(
+      users,
+      dbUsers,
+      compare: (User a, User b) => a.id.compareTo(b.id),
+      both: (User a, User b) {
+        if (a.updatedAt != b.updatedAt) {
+          toUpsert.add(a);
+          return true;
+        }
+        return false;
+      },
+      onlyFirst: (User a) => toUpsert.add(a),
+      onlySecond: (User b) => toDelete.add(b.isarId),
+    );
+    if (changes) {
+      await _db.writeTxn(() async {
+        await _db.users.deleteAll(toDelete);
+        await _db.users.putAll(toUpsert);
+      });
+    }
+    return changes;
+  }
+
+  /// Syncs remote assets owned by the logged-in user to the DB
+  /// Returns `true` if there were any changes
+  Future<bool> syncRemoteAssetsToDb(List<Asset> remote) =>
+      _lock.run(() => _syncRemoteAssetsToDb(remote));
+
+  /// Syncs remote albums to the database
+  /// returns `true` if there were any changes
+  Future<bool> syncRemoteAlbumsToDb(
+    List<AlbumResponseDto> remote, {
+    required bool isShared,
+    required FutureOr<AlbumResponseDto> Function(AlbumResponseDto) loadDetails,
+  }) =>
+      _lock.run(() => _syncRemoteAlbumsToDb(remote, isShared, loadDetails));
+
+  /// Syncs all device albums and their assets to the database
+  /// Returns `true` if there were any changes
+  Future<bool> syncLocalAlbumAssetsToDb(List<AssetPathEntity> onDevice) =>
+      _lock.run(() => _syncLocalAlbumAssetsToDb(onDevice));
+
+  /// returns all Asset IDs that are not contained in the existing list
+  List<int> sharedAssetsToRemove(
+    List<Asset> deleteCandidates,
+    List<Asset> existing,
+  ) {
+    if (deleteCandidates.isEmpty) {
+      return [];
+    }
+    deleteCandidates.sort(Asset.compareById);
+    existing.sort(Asset.compareById);
+    return _diffAssets(existing, deleteCandidates, compare: Asset.compareById)
+        .third
+        .map((e) => e.id)
+        .toList();
+  }
+
+  // private methods:
+
+  /// Syncs remote assets to the databas
+  /// returns `true` if there were any changes
+  Future<bool> _syncRemoteAssetsToDb(List<Asset> remote) async {
+    final User user = Store.get(StoreKey.currentUser);
+    final List<Asset> inDb = await _db.assets
+        .filter()
+        .ownerIdEqualTo(user.isarId)
+        .sortByDeviceId()
+        .thenByLocalId()
+        .findAll();
+    remote.sort(Asset.compareByDeviceIdLocalId);
+    final diff = _diffAssets(remote, inDb, remote: true);
+    if (diff.first.isEmpty && diff.second.isEmpty && diff.third.isEmpty) {
+      return false;
+    }
+    final idsToDelete = diff.third.map((e) => e.id).toList();
+    try {
+      await _db.writeTxn(() => _db.assets.deleteAll(idsToDelete));
+      await _upsertAssetsWithExif(diff.first + diff.second);
+    } on IsarError catch (e) {
+      debugPrint(e.toString());
+    }
+    return true;
+  }
+
+  /// Syncs remote albums to the database
+  /// returns `true` if there were any changes
+  Future<bool> _syncRemoteAlbumsToDb(
+    List<AlbumResponseDto> remote,
+    bool isShared,
+    FutureOr<AlbumResponseDto> Function(AlbumResponseDto) loadDetails,
+  ) async {
+    remote.sortBy((e) => e.id);
+
+    final baseQuery = _db.albums.where().remoteIdIsNotNull().filter();
+    final QueryBuilder<Album, Album, QAfterFilterCondition> query;
+    if (isShared) {
+      query = baseQuery.sharedEqualTo(true);
+    } else {
+      final User me = Store.get(StoreKey.currentUser);
+      query = baseQuery.owner((q) => q.isarIdEqualTo(me.isarId));
+    }
+    final List<Album> dbAlbums = await query.sortByRemoteId().findAll();
+
+    final List<Asset> toDelete = [];
+    final List<Asset> existing = [];
+
+    final bool changes = await diffSortedLists(
+      remote,
+      dbAlbums,
+      compare: (AlbumResponseDto a, Album b) => a.id.compareTo(b.remoteId!),
+      both: (AlbumResponseDto a, Album b) =>
+          _syncRemoteAlbum(a, b, toDelete, existing, loadDetails),
+      onlyFirst: (AlbumResponseDto a) =>
+          _addAlbumFromServer(a, existing, loadDetails),
+      onlySecond: (Album a) => _removeAlbumFromDb(a, toDelete),
+    );
+
+    if (isShared && toDelete.isNotEmpty) {
+      final List<int> idsToRemove = sharedAssetsToRemove(toDelete, existing);
+      if (idsToRemove.isNotEmpty) {
+        await _db.writeTxn(() => _db.assets.deleteAll(idsToRemove));
+      }
+    } else {
+      assert(toDelete.isEmpty);
+    }
+    return changes;
+  }
+
+  /// syncs albums from the server to the local database (does not support
+  /// syncing changes from local back to server)
+  /// accumulates
+  Future<bool> _syncRemoteAlbum(
+    AlbumResponseDto dto,
+    Album album,
+    List<Asset> deleteCandidates,
+    List<Asset> existing,
+    FutureOr<AlbumResponseDto> Function(AlbumResponseDto) loadDetails,
+  ) async {
+    if (!_hasAlbumResponseDtoChanged(dto, album)) {
+      return false;
+    }
+    dto = await loadDetails(dto);
+    if (dto.assetCount != dto.assets.length) {
+      return false;
+    }
+    final assetsInDb =
+        await album.assets.filter().sortByDeviceId().thenByLocalId().findAll();
+    final List<Asset> assetsOnRemote = dto.getAssets();
+    assetsOnRemote.sort(Asset.compareByDeviceIdLocalId);
+    final d = _diffAssets(assetsOnRemote, assetsInDb);
+    final List<Asset> toAdd = d.first, toUpdate = d.second, toUnlink = d.third;
+
+    // update shared users
+    final List<User> sharedUsers = album.sharedUsers.toList(growable: false);
+    sharedUsers.sort((a, b) => a.id.compareTo(b.id));
+    dto.sharedUsers.sort((a, b) => a.id.compareTo(b.id));
+    final List<String> userIdsToAdd = [];
+    final List<User> usersToUnlink = [];
+    diffSortedListsSync(
+      dto.sharedUsers,
+      sharedUsers,
+      compare: (UserResponseDto a, User b) => a.id.compareTo(b.id),
+      both: (a, b) => false,
+      onlyFirst: (UserResponseDto a) => userIdsToAdd.add(a.id),
+      onlySecond: (User a) => usersToUnlink.add(a),
+    );
+
+    // for shared album: put missing album assets into local DB
+    final resultPair = await _linkWithExistingFromDb(toAdd);
+    await _upsertAssetsWithExif(resultPair.second);
+    final assetsToLink = resultPair.first + resultPair.second;
+    final usersToLink = (await _db.users.getAllById(userIdsToAdd)).cast<User>();
+
+    album.name = dto.albumName;
+    album.shared = dto.shared;
+    album.modifiedAt = DateTime.parse(dto.updatedAt).toUtc();
+    if (album.thumbnail.value?.remoteId != dto.albumThumbnailAssetId) {
+      album.thumbnail.value = await _db.assets
+          .where()
+          .remoteIdEqualTo(dto.albumThumbnailAssetId)
+          .findFirst();
+    }
+
+    // write & commit all changes to DB
+    try {
+      await _db.writeTxn(() async {
+        await _db.assets.putAll(toUpdate);
+        await album.thumbnail.save();
+        await album.sharedUsers
+            .update(link: usersToLink, unlink: usersToUnlink);
+        await album.assets.update(link: assetsToLink, unlink: toUnlink.cast());
+        await _db.albums.put(album);
+      });
+    } on IsarError catch (e) {
+      debugPrint(e.toString());
+    }
+
+    if (album.shared || dto.shared) {
+      final userId = Store.get<User>(StoreKey.currentUser)!.isarId;
+      final foreign =
+          await album.assets.filter().not().ownerIdEqualTo(userId).findAll();
+      existing.addAll(foreign);
+
+      // delete assets in DB unless they belong to this user or part of some other shared album
+      deleteCandidates.addAll(toUnlink.where((a) => a.ownerId != userId));
+    }
+
+    return true;
+  }
+
+  /// Adds a remote album to the database while making sure to add any foreign
+  /// (shared) assets to the database beforehand
+  /// accumulates assets already existing in the database
+  Future<void> _addAlbumFromServer(
+    AlbumResponseDto dto,
+    List<Asset> existing,
+    FutureOr<AlbumResponseDto> Function(AlbumResponseDto) loadDetails,
+  ) async {
+    if (dto.assetCount != dto.assets.length) {
+      dto = await loadDetails(dto);
+    }
+    if (dto.assetCount == dto.assets.length) {
+      // in case an album contains assets not yet present in local DB:
+      // put missing album assets into local DB
+      final result = await _linkWithExistingFromDb(dto.getAssets());
+      existing.addAll(result.first);
+      await _upsertAssetsWithExif(result.second);
+
+      final Album a = await Album.remote(dto);
+      await _db.writeTxn(() => _db.albums.store(a));
+    }
+  }
+
+  /// Accumulates all suitable album assets to the `deleteCandidates` and
+  /// removes the album from the database.
+  Future<void> _removeAlbumFromDb(
+    Album album,
+    List<Asset> deleteCandidates,
+  ) async {
+    if (album.isLocal) {
+      // delete assets in DB unless they are remote or part of some other album
+      deleteCandidates.addAll(
+        await album.assets.filter().remoteIdIsNull().findAll(),
+      );
+    } else if (album.shared) {
+      final User user = Store.get(StoreKey.currentUser);
+      // delete assets in DB unless they belong to this user or are part of some other shared album
+      deleteCandidates.addAll(
+        await album.assets.filter().not().ownerIdEqualTo(user.isarId).findAll(),
+      );
+    }
+    final bool ok = await _db.writeTxn(() => _db.albums.delete(album.id));
+    assert(ok);
+  }
+
+  /// Syncs all device albums and their assets to the database
+  /// Returns `true` if there were any changes
+  Future<bool> _syncLocalAlbumAssetsToDb(List<AssetPathEntity> onDevice) async {
+    onDevice.sort((a, b) => a.id.compareTo(b.id));
+    final List<Album> inDb =
+        await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll();
+    final List<Asset> deleteCandidates = [];
+    final List<Asset> existing = [];
+    final bool anyChanges = await diffSortedLists(
+      onDevice,
+      inDb,
+      compare: (AssetPathEntity a, Album b) => a.id.compareTo(b.localId!),
+      both: (AssetPathEntity ape, Album album) =>
+          _syncAlbumInDbAndOnDevice(ape, album, deleteCandidates, existing),
+      onlyFirst: (AssetPathEntity ape) => _addAlbumFromDevice(ape, existing),
+      onlySecond: (Album a) => _removeAlbumFromDb(a, deleteCandidates),
+    );
+    final pair = _handleAssetRemoval(deleteCandidates, existing);
+    if (pair.first.isNotEmpty || pair.second.isNotEmpty) {
+      await _db.writeTxn(() async {
+        await _db.assets.deleteAll(pair.first);
+        await _db.assets.putAll(pair.second);
+      });
+    }
+    return anyChanges;
+  }
+
+  /// Syncs the device album to the album in the database
+  /// returns `true` if there were any changes
+  /// Accumulates asset candidates to delete and those already existing in DB
+  Future<bool> _syncAlbumInDbAndOnDevice(
+    AssetPathEntity ape,
+    Album album,
+    List<Asset> deleteCandidates,
+    List<Asset> existing, [
+    bool forceRefresh = false,
+  ]) async {
+    if (!forceRefresh && !await _hasAssetPathEntityChanged(ape, album)) {
+      return false;
+    }
+    if (!forceRefresh && await _syncDeviceAlbumFast(ape, album)) {
+      return true;
+    }
+
+    // general case, e.g. some assets have been deleted
+    final inDb = await album.assets.filter().sortByLocalId().findAll();
+    final List<Asset> onDevice = await ape.getAssets();
+    onDevice.sort(Asset.compareByLocalId);
+    final d = _diffAssets(onDevice, inDb, compare: Asset.compareByLocalId);
+    final List<Asset> toAdd = d.first, toUpdate = d.second, toDelete = d.third;
+    final result = await _linkWithExistingFromDb(toAdd);
+    deleteCandidates.addAll(toDelete);
+    existing.addAll(result.first);
+    album.name = ape.name;
+    album.modifiedAt = ape.lastModified!;
+    if (album.thumbnail.value != null &&
+        toDelete.contains(album.thumbnail.value)) {
+      album.thumbnail.value = null;
+    }
+    try {
+      await _db.writeTxn(() async {
+        await _db.assets.putAll(result.second);
+        await _db.assets.putAll(toUpdate);
+        await album.assets
+            .update(link: result.first + result.second, unlink: toDelete);
+        await _db.albums.put(album);
+        album.thumbnail.value ??= await album.assets.filter().findFirst();
+        await album.thumbnail.save();
+      });
+    } on IsarError catch (e) {
+      debugPrint(e.toString());
+    }
+
+    return true;
+  }
+
+  /// fast path for common case: only new assets were added to device album
+  /// returns `true` if successfull, else `false`
+  Future<bool> _syncDeviceAlbumFast(AssetPathEntity ape, Album album) async {
+    final int totalOnDevice = await ape.assetCountAsync;
+    final AssetPathEntity? modified = totalOnDevice > album.assetCount
+        ? await ape.fetchPathProperties(
+            filterOptionGroup: FilterOptionGroup(
+              updateTimeCond: DateTimeCond(
+                min: album.modifiedAt.add(const Duration(seconds: 1)),
+                max: ape.lastModified!,
+              ),
+            ),
+          )
+        : null;
+    if (modified == null) {
+      return false;
+    }
+    final List<Asset> newAssets = await modified.getAssets();
+    if (totalOnDevice != album.assets.length + newAssets.length) {
+      return false;
+    }
+    album.modifiedAt = ape.lastModified!.toUtc();
+    final result = await _linkWithExistingFromDb(newAssets);
+    try {
+      await _db.writeTxn(() async {
+        await _db.assets.putAll(result.second);
+        await album.assets.update(link: result.first + result.second);
+        await _db.albums.put(album);
+      });
+    } on IsarError catch (e) {
+      debugPrint(e.toString());
+    }
+
+    return true;
+  }
+
+  /// Adds a new album from the device to the database and Accumulates all
+  /// assets already existing in the database to the list of `existing` assets
+  Future<void> _addAlbumFromDevice(
+    AssetPathEntity ape,
+    List<Asset> existing,
+  ) async {
+    final Album a = Album.local(ape);
+    final result = await _linkWithExistingFromDb(await ape.getAssets());
+    await _upsertAssetsWithExif(result.second);
+    existing.addAll(result.first);
+    a.assets.addAll(result.first);
+    a.assets.addAll(result.second);
+    final thumb = result.first.firstOrNull ?? result.second.firstOrNull;
+    a.thumbnail.value = thumb;
+    try {
+      await _db.writeTxn(() => _db.albums.store(a));
+    } on IsarError catch (e) {
+      debugPrint(e.toString());
+    }
+  }
+
+  /// Returns a tuple (existing, updated)
+  Future<Pair<List<Asset>, List<Asset>>> _linkWithExistingFromDb(
+    List<Asset> assets,
+  ) async {
+    if (assets.isEmpty) {
+      return const Pair([], []);
+    }
+    final List<Asset> inDb = await _db.assets
+        .where()
+        .anyOf(
+          assets,
+          (q, Asset e) => q.localIdDeviceIdEqualTo(e.localId, e.deviceId),
+        )
+        .sortByDeviceId()
+        .thenByLocalId()
+        .findAll();
+    assets.sort(Asset.compareByDeviceIdLocalId);
+    final List<Asset> existing = [], toUpsert = [];
+    diffSortedListsSync(
+      inDb,
+      assets,
+      compare: Asset.compareByDeviceIdLocalId,
+      both: (Asset a, Asset b) {
+        if ((a.isLocal || !b.isLocal) &&
+            (a.isRemote || !b.isRemote) &&
+            a.updatedAt == b.updatedAt) {
+          existing.add(a);
+          return false;
+        } else {
+          toUpsert.add(b.updateFromDb(a));
+          return true;
+        }
+      },
+      onlyFirst: (Asset a) => throw Exception("programming error"),
+      onlySecond: (Asset b) => toUpsert.add(b),
+    );
+    return Pair(existing, toUpsert);
+  }
+
+  /// Inserts or updates the assets in the database with their ExifInfo (if any)
+  Future<void> _upsertAssetsWithExif(List<Asset> assets) async {
+    if (assets.isEmpty) {
+      return;
+    }
+    final exifInfos = assets.map((e) => e.exifInfo).whereNotNull().toList();
+    try {
+      await _db.writeTxn(() async {
+        await _db.assets.putAll(assets);
+        for (final Asset added in assets) {
+          added.exifInfo?.id = added.id;
+        }
+        await _db.exifInfos.putAll(exifInfos);
+      });
+    } on IsarError catch (e) {
+      debugPrint(e.toString());
+    }
+  }
+}
+
+/// Returns a triple(toAdd, toUpdate, toRemove)
+Triple<List<Asset>, List<Asset>, List<Asset>> _diffAssets(
+  List<Asset> assets,
+  List<Asset> inDb, {
+  bool? remote,
+  int Function(Asset, Asset) compare = Asset.compareByDeviceIdLocalId,
+}) {
+  final List<Asset> toAdd = [];
+  final List<Asset> toUpdate = [];
+  final List<Asset> toRemove = [];
+  diffSortedListsSync(
+    inDb,
+    assets,
+    compare: compare,
+    both: (Asset a, Asset b) {
+      if (a.updatedAt.isBefore(b.updatedAt) ||
+          (!a.isLocal && b.isLocal) ||
+          (!a.isRemote && b.isRemote)) {
+        toUpdate.add(b.updateFromDb(a));
+        debugPrint("both");
+        return true;
+      }
+      return false;
+    },
+    onlyFirst: (Asset a) {
+      if (remote == true && a.isLocal) {
+        if (a.remoteId != null) {
+          a.remoteId = null;
+          toUpdate.add(a);
+        }
+      } else if (remote == false && a.isRemote) {
+        if (a.isLocal) {
+          a.isLocal = false;
+          toUpdate.add(a);
+        }
+      } else {
+        toRemove.add(a);
+      }
+    },
+    onlySecond: (Asset b) => toAdd.add(b),
+  );
+  return Triple(toAdd, toUpdate, toRemove);
+}
+
+/// returns a tuple (toDelete toUpdate) when assets are to be deleted
+Pair<List<int>, List<Asset>> _handleAssetRemoval(
+  List<Asset> deleteCandidates,
+  List<Asset> existing,
+) {
+  if (deleteCandidates.isEmpty) {
+    return const Pair([], []);
+  }
+  deleteCandidates.sort(Asset.compareById);
+  existing.sort(Asset.compareById);
+  final triple =
+      _diffAssets(existing, deleteCandidates, compare: Asset.compareById);
+  return Pair(triple.third.map((e) => e.id).toList(), triple.second);
+}
+
+/// returns `true` if the albums differ on the surface
+Future<bool> _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async {
+  return a.name != b.name ||
+      a.lastModified != b.modifiedAt ||
+      await a.assetCountAsync != b.assetCount;
+}
+
+/// returns `true` if the albums differ on the surface
+bool _hasAlbumResponseDtoChanged(AlbumResponseDto dto, Album a) {
+  return dto.assetCount != a.assetCount ||
+      dto.albumName != a.name ||
+      dto.albumThumbnailAssetId != a.thumbnail.value?.remoteId ||
+      dto.shared != a.shared ||
+      DateTime.parse(dto.updatedAt).toUtc() != a.modifiedAt.toUtc();
+}

mobile/lib/shared/services/user.service.dart

@@ -3,24 +3,32 @@ import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:http/http.dart';
 import 'package:http_parser/http_parser.dart';
 import 'package:image_picker/image_picker.dart';
+import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/models/user.dart';
 import 'package:immich_mobile/shared/providers/api.provider.dart';
+import 'package:immich_mobile/shared/providers/db.provider.dart';
 import 'package:immich_mobile/shared/services/api.service.dart';
+import 'package:immich_mobile/shared/services/sync.service.dart';
 import 'package:immich_mobile/utils/files_helper.dart';
+import 'package:isar/isar.dart';
 import 'package:openapi/api.dart';
 
 final userServiceProvider = Provider(
   (ref) => UserService(
     ref.watch(apiServiceProvider),
+    ref.watch(dbProvider),
+    ref.watch(syncServiceProvider),
   ),
 );
 
 class UserService {
   final ApiService _apiService;
+  final Isar _db;
+  final SyncService _syncService;
 
-  UserService(this._apiService);
+  UserService(this._apiService, this._db, this._syncService);
 
-  Future<List<User>?> getAllUsers({required bool isAll}) async {
+  Future<List<User>?> _getAllUsers({required bool isAll}) async {
     try {
       final dto = await _apiService.userApi.getAllUsers(isAll);
       return dto?.map(User.fromDto).toList();
@@ -30,6 +38,14 @@ class UserService {
     }
   }
 
+  Future<List<User>> getUsersInDb({bool self = false}) async {
+    if (self) {
+      return _db.users.where().findAll();
+    }
+    final int userId = Store.get<User>(StoreKey.currentUser)!.isarId;
+    return _db.users.where().isarIdNotEqualTo(userId).findAll();
+  }
+
   Future<CreateProfileImageResponseDto?> uploadProfileImage(XFile image) async {
     try {
       var mimeType = FileHelper.getMimeType(image.path);
@@ -50,4 +66,12 @@ class UserService {
       return null;
     }
   }
+
+  Future<bool> refreshUsers() async {
+    final List<User>? users = await _getAllUsers(isAll: true);
+    if (users == null) {
+      return false;
+    }
+    return _syncService.syncUsersFromServer(users);
+  }
 }