feat(server,web): activate ETags for all API endpoints and asset serving (#1031)

This greatly reduces the network traffic by app/web.

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -14,7 +14,6 @@ import {
   Header,
   Put,
   UploadedFiles,
-  Request,
 } from '@nestjs/common';
 import { Authenticated } from '../../decorators/authenticated.decorator';
 import { AssetService } from './asset.service';
@@ -22,12 +21,12 @@ import { FileFieldsInterceptor } from '@nestjs/platform-express';
 import { assetUploadOption } from '../../config/asset-upload.config';
 import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
 import { ServeFileDto } from './dto/serve-file.dto';
-import { Response as Res, Request as Req } from 'express';
+import { Response as Res} from 'express';
 import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
 import { DeleteAssetDto } from './dto/delete-asset.dto';
 import { SearchAssetDto } from './dto/search-asset.dto';
 import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
-import { ApiBearerAuth, ApiBody, ApiConsumes, ApiHeader, ApiResponse, ApiTags } from '@nestjs/swagger';
+import { ApiBearerAuth, ApiBody, ApiConsumes, ApiHeader, ApiTags } from '@nestjs/swagger';
 import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
 import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto';
 import { AssetResponseDto } from './response-dto/asset-response.dto';
@@ -50,7 +49,6 @@ import {
   IMMICH_ARCHIVE_FILE_COUNT,
   IMMICH_CONTENT_LENGTH_HINT,
 } from '../../constants/download.constant';
-import { etag } from '../../utils/etag';
 
 @Authenticated()
 @ApiBearerAuth()
@@ -110,7 +108,7 @@ export class AssetController {
   }
 
   @Get('/file/:assetId')
-  @Header('Cache-Control', 'max-age=300')
+  @Header('Cache-Control', 'max-age=3600')
   async serveFile(
     @Headers() headers: Record<string, string>,
     @Response({ passthrough: true }) res: Res,
@@ -121,13 +119,14 @@ export class AssetController {
   }
 
   @Get('/thumbnail/:assetId')
-  @Header('Cache-Control', 'max-age=300')
+  @Header('Cache-Control', 'max-age=3600')
   async getAssetThumbnail(
+    @Headers() headers: Record<string, string>,
     @Response({ passthrough: true }) res: Res,
     @Param('assetId') assetId: string,
     @Query(new ValidationPipe({ transform: true })) query: GetAssetThumbnailDto,
   ): Promise<any> {
-    return this.assetService.getAssetThumbnail(assetId, query, res);
+    return this.assetService.getAssetThumbnail(assetId, query, res, headers);
   }
 
   @Get('/curated-objects')
@@ -176,22 +175,9 @@ export class AssetController {
     required: false,
     schema: { type: 'string' },
   })
-  @ApiResponse({
+  async getAllAssets(@GetAuthUser() authUser: AuthUserDto): Promise<AssetResponseDto[]> {
-    status: 200,
-    headers: { ETag: { required: true, schema: { type: 'string' } } },
-    type: [AssetResponseDto],
-  })
-  async getAllAssets(@GetAuthUser() authUser: AuthUserDto, @Response() response: Res, @Request() request: Req) {
     const assets = await this.assetService.getAllAssets(authUser);
-    const clientEtag = request.headers['if-none-match'];
+    return assets;
-    const json = JSON.stringify(assets);
-    const serverEtag = await etag(json);
-    response.setHeader('ETag', serverEtag);
-    if (clientEtag === serverEtag) {
-      response.status(304).end();
-    } else {
-      response.contentType('application/json').status(200).send(json);
-    }
   }
 
   @Post('/time-bucket')

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -306,7 +306,12 @@ export class AssetService {
     }
   }
 
-  public async getAssetThumbnail(assetId: string, query: GetAssetThumbnailDto, res: Res) {
+  public async getAssetThumbnail(
+    assetId: string,
+    query: GetAssetThumbnailDto,
+    res: Res,
+    headers: Record<string, string>,
+  ) {
     let fileReadStream: ReadStream;
 
     const asset = await this.assetRepository.findOne({ where: { id: assetId } });
@@ -316,28 +321,22 @@ export class AssetService {
     }
 
     try {
-      if (query.format == GetAssetThumbnailFormatEnum.JPEG) {
+      if (query.format == GetAssetThumbnailFormatEnum.WEBP && asset.webpPath && asset.webpPath.length > 0) {
+        if (await processETag(asset.webpPath, res, headers)) {
+          return;
+        }
+        await fs.access(asset.webpPath, constants.R_OK);
+        fileReadStream = createReadStream(asset.webpPath);
+      } else {
         if (!asset.resizePath) {
           throw new NotFoundException('resizePath not set');
         }
-
+        if (await processETag(asset.resizePath, res, headers)) {
-        await fs.access(asset.resizePath, constants.R_OK | constants.W_OK);
+          return;
-        fileReadStream = createReadStream(asset.resizePath);
-      } else {
-        if (asset.webpPath && asset.webpPath.length > 0) {
-          await fs.access(asset.webpPath, constants.R_OK | constants.W_OK);
-          fileReadStream = createReadStream(asset.webpPath);
-        } else {
-          if (!asset.resizePath) {
-            throw new NotFoundException('resizePath not set');
-          }
-
-          await fs.access(asset.resizePath, constants.R_OK | constants.W_OK);
-          fileReadStream = createReadStream(asset.resizePath);
         }
+        await fs.access(asset.resizePath, constants.R_OK);
+        fileReadStream = createReadStream(asset.resizePath);
       }
-
-      res.header('Cache-Control', 'max-age=300');
       return new StreamableFile(fileReadStream);
     } catch (e) {
       res.header('Cache-Control', 'none');
@@ -349,7 +348,7 @@ export class AssetService {
     }
   }
 
-  public async serveFile(assetId: string, query: ServeFileDto, res: Res, headers: any) {
+  public async serveFile(assetId: string, query: ServeFileDto, res: Res, headers: Record<string, string>) {
     let fileReadStream: ReadStream;
     const asset = await this._assetRepository.getById(assetId);
 
@@ -371,6 +370,9 @@ export class AssetService {
             Logger.error('Error serving IMAGE asset for web', 'ServeFile');
             throw new InternalServerErrorException(`Failed to serve image asset for web`, 'ServeFile');
           }
+          if (await processETag(asset.resizePath, res, headers)) {
+            return;
+          }
           await fs.access(asset.resizePath, constants.R_OK | constants.W_OK);
           fileReadStream = createReadStream(asset.resizePath);
 
@@ -384,7 +386,9 @@ export class AssetService {
           res.set({
             'Content-Type': asset.mimeType,
           });
-
+          if (await processETag(asset.originalPath, res, headers)) {
+            return;
+          }
           await fs.access(asset.originalPath, constants.R_OK | constants.W_OK);
           fileReadStream = createReadStream(asset.originalPath);
         } else {
@@ -392,7 +396,9 @@ export class AssetService {
             res.set({
               'Content-Type': 'image/webp',
             });
-
+            if (await processETag(asset.webpPath, res, headers)) {
+              return;
+            }
             await fs.access(asset.webpPath, constants.R_OK | constants.W_OK);
             fileReadStream = createReadStream(asset.webpPath);
           } else {
@@ -403,6 +409,9 @@ export class AssetService {
             if (!asset.resizePath) {
               throw new Error('resizePath not set');
             }
+            if (await processETag(asset.resizePath, res, headers)) {
+              return;
+            }
 
             await fs.access(asset.resizePath, constants.R_OK | constants.W_OK);
             fileReadStream = createReadStream(asset.resizePath);
@@ -436,9 +445,9 @@ export class AssetService {
 
         if (range) {
           /** Extracting Start and End value from Range Header */
-          let [start, end] = range.replace(/bytes=/, '').split('-');
+          const [startStr, endStr] = range.replace(/bytes=/, '').split('-');
-          start = parseInt(start, 10);
+          let start = parseInt(startStr, 10);
-          end = end ? parseInt(end, 10) : size - 1;
+          let end = endStr ? parseInt(endStr, 10) : size - 1;
 
           if (!isNaN(start) && isNaN(end)) {
             start = start;
@@ -475,7 +484,9 @@ export class AssetService {
           res.set({
             'Content-Type': mimeType,
           });
-
+          if (await processETag(asset.originalPath, res, headers)) {
+            return;
+          }
           return new StreamableFile(createReadStream(videoPath));
         }
       } catch (e) {
@@ -632,3 +643,14 @@ export class AssetService {
     return this._assetRepository.getAssetCountByUserId(authUser.id);
   }
 }
+
+async function processETag(path: string, res: Res, headers: Record<string, string>): Promise<boolean> {
+  const { size, mtimeNs } = await fs.stat(path, { bigint: true });
+  const etag = `W/"${size}-${mtimeNs}"`;
+  res.setHeader('ETag', etag);
+  if (etag === headers['if-none-match']) {
+    res.status(304);
+    return true;
+  }
+  return false;
+}

server/apps/immich/src/main.ts

@@ -14,6 +14,7 @@ async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule);
 
   app.set('trust proxy');
+  app.set('etag', 'strong');
   app.use(cookieParser());
   app.use(json({ limit: '10mb' }));
   if (process.env.NODE_ENV === 'development') {

server/apps/immich/src/types/index.d.ts

@@ -1,5 +0,0 @@
-declare module 'crypto' {
-  namespace webcrypto {
-    const subtle: SubtleCrypto;
-  }
-}

server/apps/immich/src/utils/etag.ts

@@ -1,10 +0,0 @@
-import { webcrypto } from 'node:crypto';
-const { subtle } = webcrypto;
-
-export async function etag(text: string): Promise<string> {
-    const encoder = new TextEncoder();
-    const data = encoder.encode(text);
-    const buffer = await subtle.digest('SHA-1', data);
-    const hash = Buffer.from(buffer).toString('base64').slice(0, 27);
-    return `"${data.length}-${hash}"`;
-}