KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature(mobile): hash assets & sync via checksum (#2592)

Browse Source 
* compare different sha1 implementations

* remove openssl sha1

* sync via checksum

* hash assets in batches

* hash in background, show spinner in tab

* undo tmp changes

* migrate by clearing assets

* ignore duplicate assets

* error handling

* trigger sync/merge after download and update view

* review feedback improvements

* hash in background isolate on iOS

* rework linking assets with existing from DB

* fine-grained errors on unique index violation

* hash lenth validation

* revert compute in background on iOS

* ignore duplicate assets on device

* fix bug with batching based on accumulated size

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
This commit is contained in:
Fynn Petersen-Frey
2023-06-10 20:13:59 +02:00
committed by GitHub
parent 053a0482b4
commit 73075c64d1
28 changed files with 2315 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
mobile/test/sync_service_test.dart
View File

@@ -6,32 +6,33 @@ import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/logger_message.model.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/models/user.dart';
import 'package:immich_mobile/shared/services/hash.service.dart';
import 'package:immich_mobile/shared/services/immich_logger.service.dart';
import 'package:immich_mobile/shared/services/sync.service.dart';
import 'package:isar/isar.dart';
import 'package:mockito/mockito.dart';
void main() {
Asset makeAsset({
required String localId,
required String checksum,
String? localId,
String? remoteId,
int deviceId = 1,
int ownerId = 590700560494856554, // hash of "1"
bool isLocal = false,
}) {
final DateTime date = DateTime(2000);
return Asset(
checksum: checksum,
localId: localId,
remoteId: remoteId,
deviceId: deviceId,
ownerId: ownerId,
fileCreatedAt: date,
fileModifiedAt: date,
updatedAt: date,
durationInSeconds: 0,
type: AssetType.image,
fileName: localId,
fileName: localId ?? remoteId ?? "",
isFavorite: false,
isLocal: isLocal,
isArchived: false,
);
}
@@ -53,6 +54,7 @@ void main() {
group('Test SyncService grouped', () {
late final Isar db;
final MockHashService hs = MockHashService();
final owner = User(
id: "1",
updatedAt: DateTime.now(),
@@ -71,11 +73,11 @@ void main() {
await Store.put(StoreKey.currentUser, owner);
});
final List<Asset> initialAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "1-1", isLocal: true),
makeAsset(localId: "2", isLocal: true),
makeAsset(localId: "3", isLocal: true),
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "c", localId: "1", remoteId: "1-1"),
makeAsset(checksum: "d", localId: "2"),
makeAsset(checksum: "e", localId: "3"),
];
setUp(() {
db.writeTxnSync(() {
@@ -84,11 +86,11 @@ void main() {
});
});
test('test inserting existing assets', () async {
SyncService s = SyncService(db);
SyncService s = SyncService(db, hs);
final List<Asset> remoteAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "1-1"),
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "c", remoteId: "1-1"),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
@@ -97,14 +99,14 @@ void main() {
});
test('test inserting new assets', () async {
SyncService s = SyncService(db);
SyncService s = SyncService(db, hs);
final List<Asset> remoteAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "1-1"),
makeAsset(localId: "2", remoteId: "1-2"),
makeAsset(localId: "4", remoteId: "1-4"),
makeAsset(localId: "1", remoteId: "3-1", deviceId: 3),
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "c", remoteId: "1-1"),
makeAsset(checksum: "d", remoteId: "1-2"),
makeAsset(checksum: "f", remoteId: "1-4"),
makeAsset(checksum: "g", remoteId: "3-1", deviceId: 3),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
@@ -113,14 +115,14 @@ void main() {
});
test('test syncing duplicate assets', () async {
SyncService s = SyncService(db);
SyncService s = SyncService(db, hs);
final List<Asset> remoteAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "1-1"),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "2-1b", deviceId: 2),
makeAsset(localId: "1", remoteId: "2-1c", deviceId: 2),
makeAsset(localId: "1", remoteId: "2-1d", deviceId: 2),
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "1-1"),
makeAsset(checksum: "c", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "h", remoteId: "2-1b", deviceId: 2),
makeAsset(checksum: "i", remoteId: "2-1c", deviceId: 2),
makeAsset(checksum: "j", remoteId: "2-1d", deviceId: 2),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
@@ -133,11 +135,13 @@ void main() {
final bool c3 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
expect(c3, true);
expect(db.assets.countSync(), 7);
remoteAssets.add(makeAsset(localId: "1", remoteId: "2-1e", deviceId: 2));
remoteAssets.add(makeAsset(localId: "2", remoteId: "2-2", deviceId: 2));
remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e", deviceId: 2));
remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2", deviceId: 2));
final bool c4 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
expect(c4, true);
expect(db.assets.countSync(), 9);
});
});
}
class MockHashService extends Mock implements HashService {}