Implemented Video Upload and Player (#2)

* Implementing video upload features

* setup image resize processor

* Add video thumbnail with duration and icon

* Fixed issue with video upload timeout and upper case file type on ios

* Added video player page

* Added video player page

* Fixing video player not play on ios

* Added partial file streaming for ios/android video request

* Added nginx as proxy server for better file serving

* update nginx and docker-compose file

* Video player working correctly

* Video player working correctly

* Split duration to the second

server/.dockerignore

@@ -1,3 +1,4 @@
 node_modules/
 upload/
-dist/
+dist/
+

server/docker-compose.yml

@@ -12,6 +12,8 @@ services:
     command: yarn start:dev
     ports:
       - "3000:3000"
+    # expose:
+    #   - 3000
     volumes:
       - .:/usr/src/app
       - userdata:/usr/src/app/upload
@@ -47,6 +49,21 @@ services:
     networks:
       - immich_network  
 
+  nginx:
+    container_name: proxy_nginx
+    image: nginx:latest
+    volumes:
+      - ./settings/nginx-conf:/etc/nginx/conf.d
+    ports:
+      - 2283:80
+      - 2284:443
+    logging:
+      driver: none
+    networks:
+      - immich_network
+    depends_on:
+      - server
+
 networks:
   immich_network:
 volumes:

server/package.json

@@ -61,16 +61,17 @@
     "@types/bcrypt": "^5.0.0",
     "@types/bull": "^3.15.7",
     "@types/express": "^4.17.13",
+    "@types/fluent-ffmpeg": "^2.1.20",
     "@types/imagemin": "^8.0.0",
     "@types/jest": "27.0.2",
     "@types/lodash": "^4.14.178",
     "@types/multer": "^1.4.7",
     "@types/node": "^16.0.0",
     "@types/passport-jwt": "^3.0.6",
+    "@types/sharp": "^0.29.5",
     "@types/supertest": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
-    "@types/sharp": "^0.29.5",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",

server/settings/nginx-conf/nginx.conf

@@ -0,0 +1,20 @@
+server {
+  client_max_body_size 50000M;
+
+  listen 80;
+
+  location / {
+    proxy_buffering off;
+    proxy_buffer_size 16k;
+    proxy_busy_buffers_size 24k;
+    proxy_buffers 64 4k;
+    proxy_force_ranges on;
+
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
+    proxy_pass http://immich_server:3000;
+  }
+}

server/src/api-v1/asset/asset.controller.ts

@@ -9,10 +9,10 @@ import {
   Param,
   ValidationPipe,
   StreamableFile,
-  Response,
   Query,
-  Logger,
-  UploadedFile,
+  Response,
+  Headers,
+  BadRequestException,
 } from '@nestjs/common';
 import { JwtAuthGuard } from '../../modules/immich-jwt/guards/jwt-auth.guard';
 import { AssetService } from './asset.service';
@@ -22,16 +22,22 @@ import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
 import { CreateAssetDto } from './dto/create-asset.dto';
 import { createReadStream } from 'fs';
 import { ServeFileDto } from './dto/serve-file.dto';
-import { ImageOptimizeService } from '../../modules/image-optimize/image-optimize.service';
+import { AssetOptimizeService } from '../../modules/image-optimize/image-optimize.service';
 import { AssetType } from './entities/asset.entity';
 import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
+import { Response as Res } from 'express';
+import { promisify } from 'util';
+import { stat } from 'fs';
+import { pipeline } from 'stream';
+
+const fileInfo = promisify(stat);
 
 @UseGuards(JwtAuthGuard)
 @Controller('asset')
 export class AssetController {
   constructor(
     private readonly assetService: AssetService,
-    private readonly imageOptimizeService: ImageOptimizeService,
+    private readonly assetOptimizeService: AssetOptimizeService,
   ) {}
 
   @Post('upload')
@@ -45,7 +51,11 @@ export class AssetController {
       const savedAsset = await this.assetService.createUserAsset(authUser, assetInfo, file.path, file.mimetype);
 
       if (savedAsset && savedAsset.type == AssetType.IMAGE) {
-        await this.imageOptimizeService.resizeImage(savedAsset);
+        await this.assetOptimizeService.resizeImage(savedAsset);
+      }
+
+      if (savedAsset && savedAsset.type == AssetType.VIDEO) {
+        await this.assetOptimizeService.getVideoThumbnail(savedAsset, file.originalname);
       }
     });
 
@@ -54,23 +64,81 @@ export class AssetController {
 
   @Get('/file')
   async serveFile(
+    @Headers() headers,
     @GetAuthUser() authUser: AuthUserDto,
-    @Response({ passthrough: true }) res,
+    @Response({ passthrough: true }) res: Res,
     @Query(ValidationPipe) query: ServeFileDto,
   ): Promise<StreamableFile> {
     let file = null;
     const asset = await this.assetService.findOne(authUser, query.did, query.aid);
-    res.set({
-      'Content-Type': asset.mimeType,
-    });
 
-    if (query.isThumb === 'false' || !query.isThumb) {
-      file = createReadStream(asset.originalPath);
-    } else {
-      file = createReadStream(asset.resizePath);
+    // Handle Sending Images
+    if (asset.type == AssetType.IMAGE || query.isThumb == 'true') {
+      res.set({
+        'Content-Type': asset.mimeType,
+      });
+
+      if (query.isThumb === 'false' || !query.isThumb) {
+        file = createReadStream(asset.originalPath);
+      } else {
+        file = createReadStream(asset.resizePath);
+      }
+
+      return new StreamableFile(file);
+    } else if (asset.type == AssetType.VIDEO) {
+      // Handle Handling Video
+      const { size } = await fileInfo(asset.originalPath);
+      const range = headers.range;
+
+      if (range) {
+        /** Extracting Start and End value from Range Header */
+        let [start, end] = range.replace(/bytes=/, '').split('-');
+        start = parseInt(start, 10);
+        end = end ? parseInt(end, 10) : size - 1;
+
+        if (!isNaN(start) && isNaN(end)) {
+          start = start;
+          end = size - 1;
+        }
+        if (isNaN(start) && !isNaN(end)) {
+          start = size - end;
+          end = size - 1;
+        }
+
+        // Handle unavailable range request
+        if (start >= size || end >= size) {
+          console.error('Bad Request');
+          // Return the 416 Range Not Satisfiable.
+          res.status(416).set({
+            'Content-Range': `bytes */${size}`,
+          });
+
+          throw new BadRequestException('Bad Request Range');
+        }
+
+        /** Sending Partial Content With HTTP Code 206 */
+        console.log('Sendinf file with type ', asset.mimeType);
+
+        res.status(206).set({
+          'Content-Range': `bytes ${start}-${end}/${size}`,
+          'Accept-Ranges': 'bytes',
+          'Content-Length': end - start + 1,
+          'Content-Type': asset.mimeType,
+        });
+
+        const videoStream = createReadStream(asset.originalPath, { start: start, end: end });
+
+        return new StreamableFile(videoStream);
+      } else {
+        res.set({
+          'Content-Type': asset.mimeType,
+        });
+
+        return new StreamableFile(createReadStream(asset.originalPath));
+      }
     }
 
-    return new StreamableFile(file);
+    console.log('SHOULD NOT BE HERE');
   }
 
   @Get('/all')

server/src/api-v1/asset/asset.module.ts

@@ -4,13 +4,13 @@ import { AssetController } from './asset.controller';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { AssetEntity } from './entities/asset.entity';
 import { ImageOptimizeModule } from '../../modules/image-optimize/image-optimize.module';
-import { ImageOptimizeService } from '../../modules/image-optimize/image-optimize.service';
+import { AssetOptimizeService } from '../../modules/image-optimize/image-optimize.service';
 import { BullModule } from '@nestjs/bull';
 
 @Module({
   imports: [
     BullModule.registerQueue({
-      name: 'image',
+      name: 'optimize',
       defaultJobOptions: {
         attempts: 3,
         removeOnComplete: true,
@@ -29,7 +29,7 @@ import { BullModule } from '@nestjs/bull';
     ImageOptimizeModule,
   ],
   controllers: [AssetController],
-  providers: [AssetService, ImageOptimizeService],
+  providers: [AssetService, AssetOptimizeService],
   exports: [],
 })
 export class AssetModule {}

server/src/api-v1/asset/asset.service.ts

@@ -26,9 +26,9 @@ export class AssetService {
     asset.createdAt = assetInfo.createdAt;
     asset.modifiedAt = assetInfo.modifiedAt;
     asset.isFavorite = assetInfo.isFavorite;
-    asset.lat = assetInfo.lat;
-    asset.lon = assetInfo.lon;
     asset.mimeType = mimeType;
+    asset.duration = assetInfo.duration;
+
     try {
       const res = await this.assetRepository.save(asset);
 
@@ -63,7 +63,7 @@ export class AssetService {
           lastQueryCreatedAt: query.nextPageKey || new Date().toISOString(),
         })
         .orderBy('a."createdAt"::date', 'DESC')
-        .take(10000)
+        // .take(500)
         .getMany();
 
       if (assets.length > 0) {

server/src/api-v1/asset/dto/create-asset.dto.ts

@@ -24,8 +24,5 @@ export class CreateAssetDto {
   fileExtension: string;
 
   @IsOptional()
-  lat: string;
-
-  @IsOptional()
-  lon: string;
+  duration: string;
 }

server/src/api-v1/asset/entities/asset.entity.ts

@@ -33,17 +33,11 @@ export class AssetEntity {
   @Column({ type: 'boolean', default: false })
   isFavorite: boolean;
 
-  @Column({ nullable: true })
-  description: string;
-
-  @Column({ nullable: true })
-  lat: string;
-
-  @Column({ nullable: true })
-  lon: string;
-
   @Column({ nullable: true })
   mimeType: string;
+
+  @Column({ nullable: true })
+  duration: string;
 }
 
 export enum AssetType {

server/src/main.ts

@@ -1,7 +1,6 @@
 import { NestFactory } from '@nestjs/core';
 import { NestExpressApplication } from '@nestjs/platform-express';
 import { AppModule } from './app.module';
-import ffmpeg from 'fluent-ffmpeg';
 
 async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule);

server/src/modules/image-optimize/image-optimize.module.ts

@@ -6,13 +6,13 @@ import { AssetModule } from '../../api-v1/asset/asset.module';
 import { AssetService } from '../../api-v1/asset/asset.service';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import { ImageOptimizeProcessor } from './image-optimize.processor';
-import { ImageOptimizeService } from './image-optimize.service';
+import { AssetOptimizeService } from './image-optimize.service';
 import { MachineLearningProcessor } from './machine-learning.processor';
 
 @Module({
   imports: [
     BullModule.registerQueue({
-      name: 'image',
+      name: 'optimize',
       defaultJobOptions: {
         attempts: 3,
         removeOnComplete: true,
@@ -30,7 +30,7 @@ import { MachineLearningProcessor } from './machine-learning.processor';
 
     TypeOrmModule.forFeature([AssetEntity]),
   ],
-  providers: [ImageOptimizeService, ImageOptimizeProcessor, MachineLearningProcessor],
-  exports: [ImageOptimizeService],
+  providers: [AssetOptimizeService, ImageOptimizeProcessor, MachineLearningProcessor],
+  exports: [AssetOptimizeService],
 })
 export class ImageOptimizeModule {}

server/src/modules/image-optimize/image-optimize.processor.ts

@@ -6,9 +6,10 @@ import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import sharp from 'sharp';
 import fs, { existsSync, mkdirSync } from 'fs';
 import { ConfigService } from '@nestjs/config';
-import { randomUUID } from 'crypto';
+import ffmpeg from 'fluent-ffmpeg';
+import { Logger } from '@nestjs/common';
 
-@Processor('image')
+@Processor('optimize')
 export class ImageOptimizeProcessor {
   constructor(
     @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@@ -16,8 +17,8 @@ export class ImageOptimizeProcessor {
     private configService: ConfigService,
   ) {}
 
-  @Process('optimize')
-  async handleOptimization(job: Job) {
+  @Process('resize-image')
+  async resizeUploadedImage(job: Job) {
     const { savedAsset }: { savedAsset: AssetEntity } = job.data;
 
     const basePath = this.configService.get('UPLOAD_LOCATION');
@@ -58,4 +59,32 @@ export class ImageOptimizeProcessor {
 
     return 'ok';
   }
+
+  @Process('get-video-thumbnail')
+  async resizeUploadedVideo(job: Job) {
+    const { savedAsset, filename }: { savedAsset: AssetEntity; filename: String } = job.data;
+
+    const basePath = this.configService.get('UPLOAD_LOCATION');
+    // const resizePath = savedAsset.originalPath.replace('/original/', '/thumb/');
+    console.log(filename);
+    // Create folder for thumb image if not exist
+    const resizeDir = `${basePath}/${savedAsset.userId}/thumb/${savedAsset.deviceId}`;
+
+    if (!existsSync(resizeDir)) {
+      mkdirSync(resizeDir, { recursive: true });
+    }
+
+    ffmpeg(savedAsset.originalPath)
+      .thumbnail({
+        count: 1,
+        timestamps: [1],
+        folder: resizeDir,
+        filename: `${filename}.png`,
+      })
+      .on('end', async (a) => {
+        await this.assetRepository.update(savedAsset, { resizePath: `${resizeDir}/${filename}.png` });
+      });
+
+    return 'ok';
+  }
 }

server/src/modules/image-optimize/image-optimize.service.ts

@@ -2,17 +2,15 @@ import { InjectQueue } from '@nestjs/bull';
 import { Injectable } from '@nestjs/common';
 import { Queue } from 'bull';
 import { randomUUID } from 'crypto';
-import { join } from 'path';
 import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
-import { AuthUserDto } from '../../decorators/auth-user.decorator';
 
 @Injectable()
-export class ImageOptimizeService {
-  constructor(@InjectQueue('image') private imageQueue: Queue) {}
+export class AssetOptimizeService {
+  constructor(@InjectQueue('optimize') private optimizeQueue: Queue) {}
 
   public async resizeImage(savedAsset: AssetEntity) {
-    const job = await this.imageQueue.add(
-      'optimize',
+    const job = await this.optimizeQueue.add(
+      'resize-image',
       {
         savedAsset,
       },
@@ -23,4 +21,19 @@ export class ImageOptimizeService {
       jobId: job.id,
     };
   }
+
+  public async getVideoThumbnail(savedAsset: AssetEntity, filename: String) {
+    const job = await this.optimizeQueue.add(
+      'get-video-thumbnail',
+      {
+        savedAsset,
+        filename,
+      },
+      { jobId: randomUUID() },
+    );
+
+    return {
+      jobId: job.id,
+    };
+  }
 }