KevinMidboe/immich
1
0
Fork 0
mirror of https://github.com/KevinMidboe/immich.git synced 2025-10-29 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): Add support for client-side hashing (#2072)

Browse Source 
* Modify controller DTOs

* Can check duplicates on server side

* Remove deviceassetid and deviceid

* Remove device ids from file uploader

* Add db migration for removed device ids

* Don't sanitize checksum

* Convert asset checksum to string

* Make checksum not optional for asset

* Use enums when rejecting duplicates

* Cleanup

* Return of the device id, but optional

* Don't use deviceId for upload folder

* Use checksum in thumb path

* Only use asset id in thumb path

* Openapi generation

* Put deviceAssetId back in asset response dto

* Add missing checksum in test fixture

* Add another missing checksum in test fixture

* Cleanup asset repository

* Add back previous /exists endpoint

* Require checksum to not be null

* Correctly set deviceId in db

* Remove index

* Fix compilation errors

* Make device id nullabel in asset response dto

* Reduce PR scope

* Revert asset service

* Reorder imports

* Reorder imports

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Reduce PR scope

* Update openapi

* Reduce PR scope

* refactor: asset bulk upload check

* chore: regenreate open-api

* chore: fix tests

* chore: tests

* update migrations and regenerate api

* Feat: use checksum in web file uploader

* Change to wasm-crypto

* Use crypto api for checksumming in web uploader

* Minor cleanup of file upload

* feat(web): pause and resume jobs

* Make device asset id not nullable again

* Cleanup

* Device id not nullable in response dto

* Update API specs

* Bump api specs

* Remove old TODO comment

* Remove NOT NULL constraint on checksum index

* Fix requested pubspec changes

* Remove unneeded import

* Update server/apps/immich/src/api-v1/asset/asset.service.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Update server/apps/immich/src/api-v1/asset/asset-repository.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Remove unneeded check

* Update server/apps/immich/src/api-v1/asset/asset-repository.ts

Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>

* Remove hashing in the web uploader

* Cleanup file uploader

* Remove varchar from asset entity fields

* Return 200 from bulk upload check

* Put device asset id back into asset repository

* Merge migrations

* Revert pubspec lock

* Update openapi specs

* Merge upstream changes

* Fix failing asset service tests

* Fix formatting issue

* Cleanup migrations

* Remove newline from pubspec

* Revert newline

* Checkout main version

* Revert again

* Only return AssetCheck

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
This commit is contained in:
Jonathan Jogenfors
2023-05-24 23:08:21 +02:00
committed by GitHub
parent 49b74e9091
commit 1b54c4f8e7
33 changed files with 1396 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
mobile/openapi/.openapi-generator/FILES generated
View File

@@ -17,6 +17,10 @@ doc/AlbumCountResponseDto.md
doc/AlbumResponseDto.md
doc/AllJobStatusResponseDto.md
doc/AssetApi.md
doc/AssetBulkUploadCheckDto.md
doc/AssetBulkUploadCheckItem.md
doc/AssetBulkUploadCheckResponseDto.md
doc/AssetBulkUploadCheckResult.md
doc/AssetCountByTimeBucket.md
doc/AssetCountByTimeBucketResponseDto.md
doc/AssetCountByUserIdResponseDto.md
@@ -142,6 +146,10 @@ lib/model/api_key_create_dto.dart
lib/model/api_key_create_response_dto.dart
lib/model/api_key_response_dto.dart
lib/model/api_key_update_dto.dart
lib/model/asset_bulk_upload_check_dto.dart
lib/model/asset_bulk_upload_check_item.dart
lib/model/asset_bulk_upload_check_response_dto.dart
lib/model/asset_bulk_upload_check_result.dart
lib/model/asset_count_by_time_bucket.dart
lib/model/asset_count_by_time_bucket_response_dto.dart
lib/model/asset_count_by_user_id_response_dto.dart
@@ -236,6 +244,10 @@ test/api_key_create_response_dto_test.dart
test/api_key_response_dto_test.dart
test/api_key_update_dto_test.dart
test/asset_api_test.dart
test/asset_bulk_upload_check_dto_test.dart
test/asset_bulk_upload_check_item_test.dart
test/asset_bulk_upload_check_response_dto_test.dart
test/asset_bulk_upload_check_result_test.dart
test/asset_count_by_time_bucket_response_dto_test.dart
test/asset_count_by_time_bucket_test.dart
test/asset_count_by_user_id_response_dto_test.dart

5
mobile/openapi/README.md generated
View File

@@ -90,6 +90,7 @@ Class | Method | HTTP request | Description
*AlbumApi* | [**removeUserFromAlbum**](doc//AlbumApi.md#removeuserfromalbum) | **DELETE** /album/{id}/user/{userId} |
*AlbumApi* | [**updateAlbumInfo**](doc//AlbumApi.md#updatealbuminfo) | **PATCH** /album/{id} |
*AssetApi* | [**addAssetsToSharedLink**](doc//AssetApi.md#addassetstosharedlink) | **PATCH** /asset/shared-link/add |
*AssetApi* | [**bulkUploadCheck**](doc//AssetApi.md#bulkuploadcheck) | **POST** /asset/bulk-upload-check |
*AssetApi* | [**checkDuplicateAsset**](doc//AssetApi.md#checkduplicateasset) | **POST** /asset/check |
*AssetApi* | [**checkExistingAssets**](doc//AssetApi.md#checkexistingassets) | **POST** /asset/exist |
*AssetApi* | [**createAssetsSharedLink**](doc//AssetApi.md#createassetssharedlink) | **POST** /asset/shared-link |
@@ -183,6 +184,10 @@ Class | Method | HTTP request | Description
- [AlbumCountResponseDto](doc//AlbumCountResponseDto.md)
- [AlbumResponseDto](doc//AlbumResponseDto.md)
- [AllJobStatusResponseDto](doc//AllJobStatusResponseDto.md)
- [AssetBulkUploadCheckDto](doc//AssetBulkUploadCheckDto.md)
- [AssetBulkUploadCheckItem](doc//AssetBulkUploadCheckItem.md)
- [AssetBulkUploadCheckResponseDto](doc//AssetBulkUploadCheckResponseDto.md)
- [AssetBulkUploadCheckResult](doc//AssetBulkUploadCheckResult.md)
- [AssetCountByTimeBucket](doc//AssetCountByTimeBucket.md)
- [AssetCountByTimeBucketResponseDto](doc//AssetCountByTimeBucketResponseDto.md)
- [AssetCountByUserIdResponseDto](doc//AssetCountByUserIdResponseDto.md)

58
mobile/openapi/doc/AssetApi.md generated
View File

@@ -10,6 +10,7 @@ All URIs are relative to */api*
Method | HTTP request | Description
------------- | ------------- | -------------
[**addAssetsToSharedLink**](AssetApi.md#addassetstosharedlink) | **PATCH** /asset/shared-link/add |
[**bulkUploadCheck**](AssetApi.md#bulkuploadcheck) | **POST** /asset/bulk-upload-check |
[**checkDuplicateAsset**](AssetApi.md#checkduplicateasset) | **POST** /asset/check |
[**checkExistingAssets**](AssetApi.md#checkexistingassets) | **POST** /asset/exist |
[**createAssetsSharedLink**](AssetApi.md#createassetssharedlink) | **POST** /asset/shared-link |
@@ -93,6 +94,63 @@ Name | Type | Description | Notes
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **bulkUploadCheck**
> AssetBulkUploadCheckResponseDto bulkUploadCheck(assetBulkUploadCheckDto)
Checks if assets exist by checksums
### Example
```dart
import 'package:openapi/api.dart';
// TODO Configure API key authorization: cookie
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
// TODO Configure API key authorization: api_key
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
// TODO Configure HTTP Bearer authorization: bearer
// Case 1. Use String Token
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
// Case 2. Use Function which generate token.
// String yourTokenGeneratorFunction() { ... }
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
final api_instance = AssetApi();
final assetBulkUploadCheckDto = AssetBulkUploadCheckDto(); // AssetBulkUploadCheckDto |
try {
final result = api_instance.bulkUploadCheck(assetBulkUploadCheckDto);
print(result);
} catch (e) {
print('Exception when calling AssetApi->bulkUploadCheck: $e\n');
}
```
### Parameters
Name | Type | Description | Notes
------------- | ------------- | ------------- | -------------
**assetBulkUploadCheckDto** | [**AssetBulkUploadCheckDto**](AssetBulkUploadCheckDto.md)| |
### Return type
[**AssetBulkUploadCheckResponseDto**](AssetBulkUploadCheckResponseDto.md)
### Authorization
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
### HTTP request headers
- **Content-Type**: application/json
- **Accept**: application/json
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **checkDuplicateAsset**
> CheckDuplicateAssetResponseDto checkDuplicateAsset(checkDuplicateAssetDto, key)

15
mobile/openapi/doc/AssetBulkUploadCheckDto.md generated Normal file
View File

@@ -0,0 +1,15 @@
# openapi.model.AssetBulkUploadCheckDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**assets** | [**List<AssetBulkUploadCheckItem>**](AssetBulkUploadCheckItem.md) | | [default to const []]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

16
mobile/openapi/doc/AssetBulkUploadCheckItem.md generated Normal file
View File

@@ -0,0 +1,16 @@
# openapi.model.AssetBulkUploadCheckItem
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**id** | **String** | |
**checksum** | **String** | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

15
mobile/openapi/doc/AssetBulkUploadCheckResponseDto.md generated Normal file
View File

@@ -0,0 +1,15 @@
# openapi.model.AssetBulkUploadCheckResponseDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**results** | [**List<AssetBulkUploadCheckResult>**](AssetBulkUploadCheckResult.md) | | [default to const []]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

18
mobile/openapi/doc/AssetBulkUploadCheckResult.md generated Normal file
View File

@@ -0,0 +1,18 @@
# openapi.model.AssetBulkUploadCheckResult
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**id** | **String** | |
**action** | **String** | |
**reason** | **String** | | [optional]
**assetId** | **String** | | [optional]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

4
mobile/openapi/lib/api.dart generated
View File

@@ -54,6 +54,10 @@ part 'model/admin_signup_response_dto.dart';
part 'model/album_count_response_dto.dart';
part 'model/album_response_dto.dart';
part 'model/all_job_status_response_dto.dart';
part 'model/asset_bulk_upload_check_dto.dart';
part 'model/asset_bulk_upload_check_item.dart';
part 'model/asset_bulk_upload_check_response_dto.dart';
part 'model/asset_bulk_upload_check_result.dart';
part 'model/asset_count_by_time_bucket.dart';
part 'model/asset_count_by_time_bucket_response_dto.dart';
part 'model/asset_count_by_user_id_response_dto.dart';

52
mobile/openapi/lib/api/asset_api.dart generated
View File

@@ -71,6 +71,58 @@ class AssetApi {
return null;
}
/// Checks if assets exist by checksums
///
/// Note: This method returns the HTTP [Response].
///
/// Parameters:
///
/// * [AssetBulkUploadCheckDto] assetBulkUploadCheckDto (required):
Future<Response> bulkUploadCheckWithHttpInfo(AssetBulkUploadCheckDto assetBulkUploadCheckDto,) async {
// ignore: prefer_const_declarations
final path = r'/asset/bulk-upload-check';
// ignore: prefer_final_locals
Object? postBody = assetBulkUploadCheckDto;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
const contentTypes = <String>['application/json'];
return apiClient.invokeAPI(
path,
'POST',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
/// Checks if assets exist by checksums
///
/// Parameters:
///
/// * [AssetBulkUploadCheckDto] assetBulkUploadCheckDto (required):
Future<AssetBulkUploadCheckResponseDto?> bulkUploadCheck(AssetBulkUploadCheckDto assetBulkUploadCheckDto,) async {
final response = await bulkUploadCheckWithHttpInfo(assetBulkUploadCheckDto,);
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetBulkUploadCheckResponseDto',) as AssetBulkUploadCheckResponseDto;
}
return null;
}
/// Check duplicated asset before uploading - for Web upload used
///
/// Note: This method returns the HTTP [Response].

8
mobile/openapi/lib/api_client.dart generated
View File

@@ -203,6 +203,14 @@ class ApiClient {
return AlbumResponseDto.fromJson(value);
case 'AllJobStatusResponseDto':
return AllJobStatusResponseDto.fromJson(value);
case 'AssetBulkUploadCheckDto':
return AssetBulkUploadCheckDto.fromJson(value);
case 'AssetBulkUploadCheckItem':
return AssetBulkUploadCheckItem.fromJson(value);
case 'AssetBulkUploadCheckResponseDto':
return AssetBulkUploadCheckResponseDto.fromJson(value);
case 'AssetBulkUploadCheckResult':
return AssetBulkUploadCheckResult.fromJson(value);
case 'AssetCountByTimeBucket':
return AssetCountByTimeBucket.fromJson(value);
case 'AssetCountByTimeBucketResponseDto':

109
mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart generated Normal file
View File

@@ -0,0 +1,109 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class AssetBulkUploadCheckDto {
/// Returns a new [AssetBulkUploadCheckDto] instance.
AssetBulkUploadCheckDto({
this.assets = const [],
});
List<AssetBulkUploadCheckItem> assets;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetBulkUploadCheckDto &&
other.assets == assets;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(assets.hashCode);
@override
String toString() => 'AssetBulkUploadCheckDto[assets=$assets]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'assets'] = this.assets;
return json;
}
/// Returns a new [AssetBulkUploadCheckDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static AssetBulkUploadCheckDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "AssetBulkUploadCheckDto[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "AssetBulkUploadCheckDto[$key]" has a null value in JSON.');
});
return true;
}());
return AssetBulkUploadCheckDto(
assets: AssetBulkUploadCheckItem.listFromJson(json[r'assets']),
);
}
return null;
}
static List<AssetBulkUploadCheckDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetBulkUploadCheckDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetBulkUploadCheckDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, AssetBulkUploadCheckDto> mapFromJson(dynamic json) {
final map = <String, AssetBulkUploadCheckDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = AssetBulkUploadCheckDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of AssetBulkUploadCheckDto-objects as value to a dart map
static Map<String, List<AssetBulkUploadCheckDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<AssetBulkUploadCheckDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = AssetBulkUploadCheckDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'assets',
};
}

117
mobile/openapi/lib/model/asset_bulk_upload_check_item.dart generated Normal file
View File

@@ -0,0 +1,117 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class AssetBulkUploadCheckItem {
/// Returns a new [AssetBulkUploadCheckItem] instance.
AssetBulkUploadCheckItem({
required this.id,
required this.checksum,
});
String id;
String checksum;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetBulkUploadCheckItem &&
other.id == id &&
other.checksum == checksum;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(id.hashCode) +
(checksum.hashCode);
@override
String toString() => 'AssetBulkUploadCheckItem[id=$id, checksum=$checksum]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'id'] = this.id;
json[r'checksum'] = this.checksum;
return json;
}
/// Returns a new [AssetBulkUploadCheckItem] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static AssetBulkUploadCheckItem? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "AssetBulkUploadCheckItem[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "AssetBulkUploadCheckItem[$key]" has a null value in JSON.');
});
return true;
}());
return AssetBulkUploadCheckItem(
id: mapValueOfType<String>(json, r'id')!,
checksum: mapValueOfType<String>(json, r'checksum')!,
);
}
return null;
}
static List<AssetBulkUploadCheckItem> listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetBulkUploadCheckItem>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetBulkUploadCheckItem.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, AssetBulkUploadCheckItem> mapFromJson(dynamic json) {
final map = <String, AssetBulkUploadCheckItem>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = AssetBulkUploadCheckItem.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of AssetBulkUploadCheckItem-objects as value to a dart map
static Map<String, List<AssetBulkUploadCheckItem>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<AssetBulkUploadCheckItem>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = AssetBulkUploadCheckItem.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'id',
'checksum',
};
}

109
mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart generated Normal file
View File

@@ -0,0 +1,109 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class AssetBulkUploadCheckResponseDto {
/// Returns a new [AssetBulkUploadCheckResponseDto] instance.
AssetBulkUploadCheckResponseDto({
this.results = const [],
});
List<AssetBulkUploadCheckResult> results;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetBulkUploadCheckResponseDto &&
other.results == results;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(results.hashCode);
@override
String toString() => 'AssetBulkUploadCheckResponseDto[results=$results]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'results'] = this.results;
return json;
}
/// Returns a new [AssetBulkUploadCheckResponseDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static AssetBulkUploadCheckResponseDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "AssetBulkUploadCheckResponseDto[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "AssetBulkUploadCheckResponseDto[$key]" has a null value in JSON.');
});
return true;
}());
return AssetBulkUploadCheckResponseDto(
results: AssetBulkUploadCheckResult.listFromJson(json[r'results']),
);
}
return null;
}
static List<AssetBulkUploadCheckResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetBulkUploadCheckResponseDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetBulkUploadCheckResponseDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, AssetBulkUploadCheckResponseDto> mapFromJson(dynamic json) {
final map = <String, AssetBulkUploadCheckResponseDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = AssetBulkUploadCheckResponseDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of AssetBulkUploadCheckResponseDto-objects as value to a dart map
static Map<String, List<AssetBulkUploadCheckResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<AssetBulkUploadCheckResponseDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = AssetBulkUploadCheckResponseDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'results',
};
}

293
mobile/openapi/lib/model/asset_bulk_upload_check_result.dart generated Normal file
View File

@@ -0,0 +1,293 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class AssetBulkUploadCheckResult {
/// Returns a new [AssetBulkUploadCheckResult] instance.
AssetBulkUploadCheckResult({
required this.id,
required this.action,
this.reason,
this.assetId,
});
String id;
AssetBulkUploadCheckResultActionEnum action;
AssetBulkUploadCheckResultReasonEnum? reason;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
String? assetId;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetBulkUploadCheckResult &&
other.id == id &&
other.action == action &&
other.reason == reason &&
other.assetId == assetId;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(id.hashCode) +
(action.hashCode) +
(reason == null ? 0 : reason!.hashCode) +
(assetId == null ? 0 : assetId!.hashCode);
@override
String toString() => 'AssetBulkUploadCheckResult[id=$id, action=$action, reason=$reason, assetId=$assetId]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'id'] = this.id;
json[r'action'] = this.action;
if (this.reason != null) {
json[r'reason'] = this.reason;
} else {
// json[r'reason'] = null;
}
if (this.assetId != null) {
json[r'assetId'] = this.assetId;
} else {
// json[r'assetId'] = null;
}
return json;
}
/// Returns a new [AssetBulkUploadCheckResult] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static AssetBulkUploadCheckResult? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "AssetBulkUploadCheckResult[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "AssetBulkUploadCheckResult[$key]" has a null value in JSON.');
});
return true;
}());
return AssetBulkUploadCheckResult(
id: mapValueOfType<String>(json, r'id')!,
action: AssetBulkUploadCheckResultActionEnum.fromJson(json[r'action'])!,
reason: AssetBulkUploadCheckResultReasonEnum.fromJson(json[r'reason']),
assetId: mapValueOfType<String>(json, r'assetId'),
);
}
return null;
}
static List<AssetBulkUploadCheckResult> listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetBulkUploadCheckResult>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetBulkUploadCheckResult.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, AssetBulkUploadCheckResult> mapFromJson(dynamic json) {
final map = <String, AssetBulkUploadCheckResult>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = AssetBulkUploadCheckResult.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of AssetBulkUploadCheckResult-objects as value to a dart map
static Map<String, List<AssetBulkUploadCheckResult>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<AssetBulkUploadCheckResult>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = AssetBulkUploadCheckResult.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'id',
'action',
};
}
class AssetBulkUploadCheckResultActionEnum {
/// Instantiate a new enum with the provided [value].
const AssetBulkUploadCheckResultActionEnum._(this.value);
/// The underlying value of this enum member.
final String value;
@override
String toString() => value;
String toJson() => value;
static const accept = AssetBulkUploadCheckResultActionEnum._(r'accept');
static const reject = AssetBulkUploadCheckResultActionEnum._(r'reject');
/// List of all possible values in this [enum][AssetBulkUploadCheckResultActionEnum].
static const values = <AssetBulkUploadCheckResultActionEnum>[
accept,
reject,
];
static AssetBulkUploadCheckResultActionEnum? fromJson(dynamic value) => AssetBulkUploadCheckResultActionEnumTypeTransformer().decode(value);
static List<AssetBulkUploadCheckResultActionEnum>? listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetBulkUploadCheckResultActionEnum>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetBulkUploadCheckResultActionEnum.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
}
/// Transformation class that can [encode] an instance of [AssetBulkUploadCheckResultActionEnum] to String,
/// and [decode] dynamic data back to [AssetBulkUploadCheckResultActionEnum].
class AssetBulkUploadCheckResultActionEnumTypeTransformer {
factory AssetBulkUploadCheckResultActionEnumTypeTransformer() => _instance ??= const AssetBulkUploadCheckResultActionEnumTypeTransformer._();
const AssetBulkUploadCheckResultActionEnumTypeTransformer._();
String encode(AssetBulkUploadCheckResultActionEnum data) => data.value;
/// Decodes a [dynamic value][data] to a AssetBulkUploadCheckResultActionEnum.
///
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
///
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
/// and users are still using an old app with the old code.
AssetBulkUploadCheckResultActionEnum? decode(dynamic data, {bool allowNull = true}) {
if (data != null) {
switch (data) {
case r'accept': return AssetBulkUploadCheckResultActionEnum.accept;
case r'reject': return AssetBulkUploadCheckResultActionEnum.reject;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');
}
}
}
return null;
}
/// Singleton [AssetBulkUploadCheckResultActionEnumTypeTransformer] instance.
static AssetBulkUploadCheckResultActionEnumTypeTransformer? _instance;
}
class AssetBulkUploadCheckResultReasonEnum {
/// Instantiate a new enum with the provided [value].
const AssetBulkUploadCheckResultReasonEnum._(this.value);
/// The underlying value of this enum member.
final String value;
@override
String toString() => value;
String toJson() => value;
static const duplicate = AssetBulkUploadCheckResultReasonEnum._(r'duplicate');
static const unsupportedFormat = AssetBulkUploadCheckResultReasonEnum._(r'unsupported-format');
/// List of all possible values in this [enum][AssetBulkUploadCheckResultReasonEnum].
static const values = <AssetBulkUploadCheckResultReasonEnum>[
duplicate,
unsupportedFormat,
];
static AssetBulkUploadCheckResultReasonEnum? fromJson(dynamic value) => AssetBulkUploadCheckResultReasonEnumTypeTransformer().decode(value);
static List<AssetBulkUploadCheckResultReasonEnum>? listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetBulkUploadCheckResultReasonEnum>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetBulkUploadCheckResultReasonEnum.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
}
/// Transformation class that can [encode] an instance of [AssetBulkUploadCheckResultReasonEnum] to String,
/// and [decode] dynamic data back to [AssetBulkUploadCheckResultReasonEnum].
class AssetBulkUploadCheckResultReasonEnumTypeTransformer {
factory AssetBulkUploadCheckResultReasonEnumTypeTransformer() => _instance ??= const AssetBulkUploadCheckResultReasonEnumTypeTransformer._();
const AssetBulkUploadCheckResultReasonEnumTypeTransformer._();
String encode(AssetBulkUploadCheckResultReasonEnum data) => data.value;
/// Decodes a [dynamic value][data] to a AssetBulkUploadCheckResultReasonEnum.
///
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
///
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
/// and users are still using an old app with the old code.
AssetBulkUploadCheckResultReasonEnum? decode(dynamic data, {bool allowNull = true}) {
if (data != null) {
switch (data) {
case r'duplicate': return AssetBulkUploadCheckResultReasonEnum.duplicate;
case r'unsupported-format': return AssetBulkUploadCheckResultReasonEnum.unsupportedFormat;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');
}
}
}
return null;
}
/// Singleton [AssetBulkUploadCheckResultReasonEnumTypeTransformer] instance.
static AssetBulkUploadCheckResultReasonEnumTypeTransformer? _instance;
}

7
mobile/openapi/test/asset_api_test.dart generated
View File

@@ -22,6 +22,13 @@ void main() {
// TODO
});
// Checks if assets exist by checksums
//
//Future<AssetBulkUploadCheckResponseDto> bulkUploadCheck(AssetBulkUploadCheckDto assetBulkUploadCheckDto) async
test('test bulkUploadCheck', () async {
// TODO
});
// Check duplicated asset before uploading - for Web upload used
//
//Future<CheckDuplicateAssetResponseDto> checkDuplicateAsset(CheckDuplicateAssetDto checkDuplicateAssetDto, { String key }) async

27
mobile/openapi/test/asset_bulk_upload_check_dto_test.dart generated Normal file
View File

@@ -0,0 +1,27 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for AssetBulkUploadCheckDto
void main() {
// final instance = AssetBulkUploadCheckDto();
group('test AssetBulkUploadCheckDto', () {
// List<AssetBulkUploadCheckItem> assets (default value: const [])
test('to test the property `assets`', () async {
// TODO
});
});
}

32
mobile/openapi/test/asset_bulk_upload_check_item_test.dart generated Normal file
View File

@@ -0,0 +1,32 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for AssetBulkUploadCheckItem
void main() {
// final instance = AssetBulkUploadCheckItem();
group('test AssetBulkUploadCheckItem', () {
// String id
test('to test the property `id`', () async {
// TODO
});
// String checksum
test('to test the property `checksum`', () async {
// TODO
});
});
}

27
mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart generated Normal file
View File

@@ -0,0 +1,27 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for AssetBulkUploadCheckResponseDto
void main() {
// final instance = AssetBulkUploadCheckResponseDto();
group('test AssetBulkUploadCheckResponseDto', () {
// List<AssetBulkUploadCheckResult> results (default value: const [])
test('to test the property `results`', () async {
// TODO
});
});
}

42
mobile/openapi/test/asset_bulk_upload_check_result_test.dart generated Normal file
View File

@@ -0,0 +1,42 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for AssetBulkUploadCheckResult
void main() {
// final instance = AssetBulkUploadCheckResult();
group('test AssetBulkUploadCheckResult', () {
// String id
test('to test the property `id`', () async {
// TODO
});
// String action
test('to test the property `action`', () async {
// TODO
});
// String reason
test('to test the property `reason`', () async {
// TODO
});
// String assetId
test('to test the property `assetId`', () async {
// TODO
});
});
}

45
server/apps/immich/src/api-v1/asset/asset-repository.ts
View File

@@ -10,13 +10,17 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
import { In } from 'typeorm/find-options/operator/In';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { ITagRepository } from '../tag/tag.repository';
import { IsNull, Not } from 'typeorm';
import { AssetSearchDto } from './dto/asset-search.dto';
export interface AssetCheck {
id: string;
checksum: Buffer;
}
export interface IAssetRepository {
get(id: string): Promise<AssetEntity | null>;
create(
@@ -38,11 +42,8 @@ export interface IAssetRepository {
getAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
getArchivedAssetCountByUserId(userId: string): Promise<AssetCountByUserIdResponseDto>;
getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
getExistingAssets(
userId: string,
checkDuplicateAssetDto: CheckExistingAssetsDto,
): Promise<CheckExistingAssetsResponseDto>;
getAssetsByChecksums(userId: string, checksums: Buffer[]): Promise<AssetCheck[]>;
getExistingAssets(userId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise<string[]>;
countByIdAndUser(assetId: string, userId: string): Promise<number>;
}
@@ -310,41 +311,39 @@ export class AssetRepository implements IAssetRepository {
* @returns Promise<string[]> - Array of assetIds belong to the device
*/
async getAllByDeviceId(ownerId: string, deviceId: string): Promise<string[]> {
const rows = await this.assetRepository.find({
const items = await this.assetRepository.find({
select: { deviceAssetId: true },
where: {
ownerId,
deviceId,
isVisible: true,
},
select: ['deviceAssetId'],
});
const res: string[] = [];
rows.forEach((v) => res.push(v.deviceAssetId));
return res;
return items.map((asset) => asset.deviceAssetId);
}
/**
* Get asset by checksum on the database
* Get assets by checksums on the database
* @param ownerId
* @param checksum
* @param checksums
*
*/
getAssetByChecksum(ownerId: string, checksum: Buffer): Promise<AssetEntity> {
return this.assetRepository.findOneOrFail({
async getAssetsByChecksums(ownerId: string, checksums: Buffer[]): Promise<AssetCheck[]> {
return this.assetRepository.find({
select: {
id: true,
checksum: true,
},
where: {
ownerId,
checksum,
checksum: In(checksums),
},
relations: ['exifInfo'],
});
}
async getExistingAssets(
ownerId: string,
checkDuplicateAssetDto: CheckExistingAssetsDto,
): Promise<CheckExistingAssetsResponseDto> {
const existingAssets = await this.assetRepository.find({
async getExistingAssets(ownerId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise<string[]> {
const assets = await this.assetRepository.find({
select: { deviceAssetId: true },
where: {
deviceAssetId: In(checkDuplicateAssetDto.deviceAssetIds),
@@ -352,7 +351,7 @@ export class AssetRepository implements IAssetRepository {
ownerId,
},
});
return new CheckExistingAssetsResponseDto(existingAssets.map((a) => a.deviceAssetId));
return assets.map((asset) => asset.deviceAssetId);
}
async countByIdAndUser(assetId: string, ownerId: string): Promise<number> {

15
server/apps/immich/src/api-v1/asset/asset.controller.ts
View File

@@ -57,6 +57,8 @@ import { AssetSearchDto } from './dto/asset-search.dto';
import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config';
import FileNotEmptyValidator from '../validation/file-not-empty-validator';
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
import { AssetBulkUploadCheckResponseDto } from './response-dto/asset-check-response.dto';
import { AssetIdDto } from './dto/asset-id.dto';
import { DeviceIdDto } from './dto/device-id.dto';
@@ -332,6 +334,19 @@ export class AssetController {
return await this.assetService.checkExistingAssets(authUser, checkExistingAssetsDto);
}
/**
* Checks if assets exist by checksums
*/
@Authenticated()
@Post('/bulk-upload-check')
@HttpCode(200)
bulkUploadCheck(
@GetAuthUser() authUser: AuthUserDto,
@Body(ValidationPipe) dto: AssetBulkUploadCheckDto,
): Promise<AssetBulkUploadCheckResponseDto> {
return this.assetService.bulkUploadCheck(authUser, dto);
}
@Authenticated()
@Post('/shared-link')
async createAssetsSharedLink(

2
server/apps/immich/src/api-v1/asset/asset.core.ts
View File

@@ -17,7 +17,7 @@ export class AssetCore {
owner: { id: authUser.id } as UserEntity,
mimeType: file.mimeType,
checksum: file.checksum || null,
checksum: file.checksum,
originalPath: file.originalPath,
deviceAssetId: dto.deviceAssetId,

4
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@@ -157,7 +157,7 @@ describe('AssetService', () => {
getLocationsByUserId: jest.fn(),
getSearchPropertiesByUserId: jest.fn(),
getAssetByTimeBucket: jest.fn(),
getAssetByChecksum: jest.fn(),
getAssetsByChecksums: jest.fn(),
getAssetCountByUserId: jest.fn(),
getArchivedAssetCountByUserId: jest.fn(),
getExistingAssets: jest.fn(),
@@ -299,7 +299,7 @@ describe('AssetService', () => {
(error as any).constraint = 'UQ_userid_checksum';
assetRepositoryMock.create.mockRejectedValue(error);
assetRepositoryMock.getAssetByChecksum.mockResolvedValue(_getAsset_1());
assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([_getAsset_1()]);
await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' });

48
server/apps/immich/src/api-v1/asset/asset.service.ts
View File

@@ -63,6 +63,12 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
import { AssetSearchDto } from './dto/asset-search.dto';
import { AddAssetsDto } from '../album/dto/add-assets.dto';
import { RemoveAssetsDto } from '../album/dto/remove-assets.dto';
import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
import {
AssetUploadAction,
AssetRejectReason,
AssetBulkUploadCheckResponseDto,
} from './response-dto/asset-check-response.dto';
const fileInfo = promisify(stat);
@@ -128,7 +134,8 @@ export class AssetService {
// handle duplicates with a success response
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
const duplicate = await this.getAssetByChecksum(authUser.id, file.checksum);
const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum);
const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
return { id: duplicate.id, duplicate: true };
}
@@ -463,7 +470,40 @@ export class AssetService {
authUser: AuthUserDto,
checkExistingAssetsDto: CheckExistingAssetsDto,
): Promise<CheckExistingAssetsResponseDto> {
return this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto);
return {
existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto),
};
}
async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
const resultsMap: Record<string, string> = {};
for (const { id, checksum } of results) {
resultsMap[checksum.toString('hex')] = id;
}
return {
results: dto.assets.map(({ id, checksum }) => {
const duplicate = resultsMap[checksum];
if (duplicate) {
return {
id,
assetId: duplicate,
action: AssetUploadAction.REJECT,
reason: AssetRejectReason.DUPLICATE,
};
}
// TODO mime-check
return {
id,
action: AssetUploadAction.ACCEPT,
};
}),
};
}
async getAssetCountByTimeBucket(
@@ -482,10 +522,6 @@ export class AssetService {
return mapAssetCountByTimeBucket(result);
}
getAssetByChecksum(userId: string, checksum: Buffer) {
return this._assetRepository.getAssetByChecksum(userId, checksum);
}
getAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
return this._assetRepository.getAssetCountByUserId(authUser.id);
}

19
server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts Normal file
View File

@@ -0,0 +1,19 @@
import { Type } from 'class-transformer';
import { IsArray, IsNotEmpty, IsString, ValidateNested } from 'class-validator';
export class AssetBulkUploadCheckItem {
@IsString()
@IsNotEmpty()
id!: string;
@IsString()
@IsNotEmpty()
checksum!: string;
}
export class AssetBulkUploadCheckDto {
@IsArray()
@ValidateNested({ each: true })
@Type(() => AssetBulkUploadCheckItem)
assets!: AssetBulkUploadCheckItem[];
}

20
server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts Normal file
View File

@@ -0,0 +1,20 @@
export class AssetBulkUploadCheckResult {
id!: string;
action!: AssetUploadAction;
reason?: AssetRejectReason;
assetId?: string;
}
export class AssetBulkUploadCheckResponseDto {
results!: AssetBulkUploadCheckResult[];
}
export enum AssetUploadAction {
ACCEPT = 'accept',
REJECT = 'reject',
}
export enum AssetRejectReason {
DUPLICATE = 'duplicate',
UNSUPPORTED_FORMAT = 'unsupported-format',
}

5
server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts
View File

@@ -1,6 +1,3 @@
export class CheckExistingAssetsResponseDto {
constructor(existingIds: string[]) {
this.existingIds = existingIds;
}
existingIds: string[];
existingIds!: string[];
}

115
server/immich-openapi-specs.json
View File

@@ -3251,6 +3251,49 @@
]
}
},
"/asset/bulk-upload-check": {
"post": {
"operationId": "bulkUploadCheck",
"description": "Checks if assets exist by checksums",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AssetBulkUploadCheckDto"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AssetBulkUploadCheckResponseDto"
}
}
}
}
},
"tags": [
"Asset"
],
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
]
}
},
"/asset/shared-link": {
"post": {
"operationId": "createAssetsSharedLink",
@@ -6046,6 +6089,78 @@
"existingIds"
]
},
"AssetBulkUploadCheckItem": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"checksum": {
"type": "string"
}
},
"required": [
"id",
"checksum"
]
},
"AssetBulkUploadCheckDto": {
"type": "object",
"properties": {
"assets": {
"type": "array",
"items": {
"$ref": "#/components/schemas/AssetBulkUploadCheckItem"
}
}
},
"required": [
"assets"
]
},
"AssetBulkUploadCheckResult": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"action": {
"type": "string",
"enum": [
"accept",
"reject"
]
},
"reason": {
"type": "string",
"enum": [
"duplicate",
"unsupported-format"
]
},
"assetId": {
"type": "string"
}
},
"required": [
"id",
"action"
]
},
"AssetBulkUploadCheckResponseDto": {
"type": "object",
"properties": {
"results": {
"type": "array",
"items": {
"$ref": "#/components/schemas/AssetBulkUploadCheckResult"
}
}
},
"required": [
"results"
]
},
"CreateAssetsShareLinkDto": {
"type": "object",
"properties": {

5
server/libs/domain/test/fixtures.ts
View File

@@ -147,6 +147,7 @@ export const assetEntityStub = {
deviceId: 'device-id',
originalPath: 'upload/upload/path.ext',
resizePath: null,
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.IMAGE,
webpPath: null,
encodedVideoPath: null,
@@ -173,6 +174,7 @@ export const assetEntityStub = {
deviceId: 'device-id',
originalPath: '/original/path.ext',
resizePath: '/uploads/user-id/thumbs/path.ext',
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.IMAGE,
webpPath: null,
encodedVideoPath: null,
@@ -201,6 +203,7 @@ export const assetEntityStub = {
deviceId: 'device-id',
originalPath: '/original/path.ext',
resizePath: '/uploads/user-id/thumbs/path.ext',
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.VIDEO,
webpPath: null,
encodedVideoPath: null,
@@ -246,6 +249,7 @@ export const assetEntityStub = {
owner: userEntityStub.user1,
ownerId: 'user-id',
deviceId: 'device-id',
checksum: Buffer.from('file hash', 'utf8'),
originalPath: '/original/path.ext',
resizePath: '/uploads/user-id/thumbs/path.ext',
type: AssetType.IMAGE,
@@ -663,6 +667,7 @@ export const sharedLinkStub = {
type: AssetType.VIDEO,
originalPath: 'fake_path/jpeg',
resizePath: '',
checksum: Buffer.from('file hash', 'utf8'),
fileModifiedAt: today.toISOString(),
fileCreatedAt: today.toISOString(),
createdAt: today.toISOString(),

6
server/libs/infra/src/entities/asset.entity.ts
View File

@@ -75,9 +75,9 @@ export class AssetEntity {
@Column({ type: 'varchar', nullable: true })
mimeType!: string | null;
@Column({ type: 'bytea', nullable: true, select: false })
@Index({ where: `'checksum' IS NOT NULL` }) // avoid null index
checksum?: Buffer | null; // sha1 checksum
@Column({ type: 'bytea' })
@Index()
checksum!: Buffer; // sha1 checksum
@Column({ type: 'varchar', nullable: true })
duration!: string | null;

19
server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts Normal file
View File

@@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class RequireChecksumNotNull1684328185099 implements MigrationInterface {
name = 'removeNotNullFromChecksumIndex1684328185099';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`);
await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" SET NOT NULL`);
await queryRunner.query(`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum") `);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "public"."IDX_8d3efe36c0755849395e6ea866"`);
await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" DROP NOT NULL`);
await queryRunner.query(
`CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE ('checksum' IS NOT NULL)`,
);
}
}

164
web/src/api/open-api/api.ts generated
View File

@@ -346,6 +346,96 @@ export interface AllJobStatusResponseDto {
*/
'recognize-faces-queue': JobStatusDto;
}
/**
*
* @export
* @interface AssetBulkUploadCheckDto
*/
export interface AssetBulkUploadCheckDto {
/**
*
* @type {Array<AssetBulkUploadCheckItem>}
* @memberof AssetBulkUploadCheckDto
*/
'assets': Array<AssetBulkUploadCheckItem>;
}
/**
*
* @export
* @interface AssetBulkUploadCheckItem
*/
export interface AssetBulkUploadCheckItem {
/**
*
* @type {string}
* @memberof AssetBulkUploadCheckItem
*/
'id': string;
/**
*
* @type {string}
* @memberof AssetBulkUploadCheckItem
*/
'checksum': string;
}
/**
*
* @export
* @interface AssetBulkUploadCheckResponseDto
*/
export interface AssetBulkUploadCheckResponseDto {
/**
*
* @type {Array<AssetBulkUploadCheckResult>}
* @memberof AssetBulkUploadCheckResponseDto
*/
'results': Array<AssetBulkUploadCheckResult>;
}
/**
*
* @export
* @interface AssetBulkUploadCheckResult
*/
export interface AssetBulkUploadCheckResult {
/**
*
* @type {string}
* @memberof AssetBulkUploadCheckResult
*/
'id': string;
/**
*
* @type {string}
* @memberof AssetBulkUploadCheckResult
*/
'action': AssetBulkUploadCheckResultActionEnum;
/**
*
* @type {string}
* @memberof AssetBulkUploadCheckResult
*/
'reason'?: AssetBulkUploadCheckResultReasonEnum;
/**
*
* @type {string}
* @memberof AssetBulkUploadCheckResult
*/
'assetId'?: string;
}
export const AssetBulkUploadCheckResultActionEnum = {
Accept: 'accept',
Reject: 'reject'
} as const;
export type AssetBulkUploadCheckResultActionEnum = typeof AssetBulkUploadCheckResultActionEnum[keyof typeof AssetBulkUploadCheckResultActionEnum];
export const AssetBulkUploadCheckResultReasonEnum = {
Duplicate: 'duplicate',
UnsupportedFormat: 'unsupported-format'
} as const;
export type AssetBulkUploadCheckResultReasonEnum = typeof AssetBulkUploadCheckResultReasonEnum[keyof typeof AssetBulkUploadCheckResultReasonEnum];
/**
*
* @export
@@ -4120,6 +4210,50 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
options: localVarRequestOptions,
};
},
/**
* Checks if assets exist by checksums
* @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
bulkUploadCheck: async (assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
// verify required parameter 'assetBulkUploadCheckDto' is not null or undefined
assertParamExists('bulkUploadCheck', 'assetBulkUploadCheckDto', assetBulkUploadCheckDto)
const localVarPath = `/asset/bulk-upload-check`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
let baseOptions;
if (configuration) {
baseOptions = configuration.baseOptions;
}
const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options};
const localVarHeaderParameter = {} as any;
const localVarQueryParameter = {} as any;
// authentication cookie required
// authentication api_key required
await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration)
// authentication bearer required
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
localVarHeaderParameter['Content-Type'] = 'application/json';
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
localVarRequestOptions.data = serializeDataIfNeeded(assetBulkUploadCheckDto, localVarRequestOptions, configuration)
return {
url: toPathString(localVarUrlObj),
options: localVarRequestOptions,
};
},
/**
* Check duplicated asset before uploading - for Web upload used
* @param {CheckDuplicateAssetDto} checkDuplicateAssetDto
@@ -5312,6 +5446,16 @@ export const AssetApiFp = function(configuration?: Configuration) {
const localVarAxiosArgs = await localVarAxiosParamCreator.addAssetsToSharedLink(addAssetsDto, key, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
* Checks if assets exist by checksums
* @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<AssetBulkUploadCheckResponseDto>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.bulkUploadCheck(assetBulkUploadCheckDto, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
* Check duplicated asset before uploading - for Web upload used
* @param {CheckDuplicateAssetDto} checkDuplicateAssetDto
@@ -5595,6 +5739,15 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
addAssetsToSharedLink(addAssetsDto: AddAssetsDto, key?: string, options?: any): AxiosPromise<SharedLinkResponseDto> {
return localVarFp.addAssetsToSharedLink(addAssetsDto, key, options).then((request) => request(axios, basePath));
},
/**
* Checks if assets exist by checksums
* @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: any): AxiosPromise<AssetBulkUploadCheckResponseDto> {
return localVarFp.bulkUploadCheck(assetBulkUploadCheckDto, options).then((request) => request(axios, basePath));
},
/**
* Check duplicated asset before uploading - for Web upload used
* @param {CheckDuplicateAssetDto} checkDuplicateAssetDto
@@ -5856,6 +6009,17 @@ export class AssetApi extends BaseAPI {
return AssetApiFp(this.configuration).addAssetsToSharedLink(addAssetsDto, key, options).then((request) => request(this.axios, this.basePath));
}
/**
* Checks if assets exist by checksums
* @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof AssetApi
*/
public bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: AxiosRequestConfig) {
return AssetApiFp(this.configuration).bulkUploadCheck(assetBulkUploadCheckDto, options).then((request) => request(this.axios, this.basePath));
}
/**
* Check duplicated asset before uploading - for Web upload used
* @param {CheckDuplicateAssetDto} checkDuplicateAssetDto

21
web/src/lib/utils/file-uploader.ts
View File

@@ -4,7 +4,7 @@ import {
} from './../components/shared-components/notification/notification';
import { uploadAssetsStore } from '$lib/stores/upload';
import type { UploadAsset } from '../models/upload-asset';
import { api, AssetFileUploadResponseDto } from '@api';
import { AssetFileUploadResponseDto } from '@api';
import { addAssetsToAlbum, getFileMimeType, getFilenameExtension } from '$lib/utils/asset-utils';
import { mergeMap, filter, firstValueFrom, from, of, combineLatestAll } from 'rxjs';
import axios from 'axios';
@@ -73,7 +73,7 @@ async function fileUploader(
const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified;
try {
// Create and add Unique ID of asset on the device
// Create and add pseudo-unique ID of asset on the device
formData.append('deviceAssetId', deviceAssetId);
// Get device id - for web -> use WEB
@@ -102,23 +102,6 @@ async function fileUploader(
// failed uploads.
formData.append('assetData', new File([asset], asset.name, { type: mimeType }));
// Check if asset upload on server before performing upload
const { data, status } = await api.assetApi.checkDuplicateAsset(
{
deviceAssetId: String(deviceAssetId),
deviceId: 'WEB'
},
sharedKey
);
if (status === 200 && data.isExist && data.id) {
if (albumId) {
await addAssetsToAlbum(albumId, [data.id], sharedKey);
}
return data.id;
}
const newUploadAsset: UploadAsset = {
id: deviceAssetId,
file: asset,