feat(server): CLIP search integration (#1939)

machine-learning/src/main.py

@@ -1,43 +1,58 @@
 import os
 from flask import Flask, request
 from transformers import pipeline
+from sentence_transformers import SentenceTransformer, util
+from PIL import Image
 
+is_dev = os.getenv('NODE_ENV') == 'development'
+server_port = os.getenv('MACHINE_LEARNING_PORT', 3003)
+server_host = os.getenv('MACHINE_LEARNING_HOST', '0.0.0.0')
+
+classification_model = os.getenv('MACHINE_LEARNING_CLASSIFICATION_MODEL', 'microsoft/resnet-50')
+object_model = os.getenv('MACHINE_LEARNING_OBJECT_MODEL', 'hustvl/yolos-tiny')
+clip_image_model = os.getenv('MACHINE_LEARNING_CLIP_IMAGE_MODEL', 'clip-ViT-B-32')
+clip_text_model = os.getenv('MACHINE_LEARNING_CLIP_TEXT_MODEL', 'clip-ViT-B-32')
+
+_model_cache = {}
+def _get_model(model, task=None):
+  global _model_cache
+  key = '|'.join([model, str(task)])
+  if key not in _model_cache:
+    if task:
+      _model_cache[key] = pipeline(model=model, task=task)
+    else:
+      _model_cache[key] = SentenceTransformer(model)
+  return _model_cache[key]
 
 server = Flask(__name__)
 
-
-classifier = pipeline(
-    task="image-classification",
-    model="microsoft/resnet-50"
-)
-
-detector = pipeline(
-    task="object-detection",
-    model="hustvl/yolos-tiny"
-)
-
-
-# Environment resolver
-is_dev = os.getenv('NODE_ENV') == 'development'
-server_port = os.getenv('MACHINE_LEARNING_PORT') or 3003
-
-
 @server.route("/ping")
 def ping():
     return "pong"
 
-
 @server.route("/object-detection/detect-object", methods=['POST'])
 def object_detection():
+    model = _get_model(object_model, 'object-detection')
     assetPath = request.json['thumbnailPath']
-    return run_engine(detector, assetPath), 201
-
+    return run_engine(model, assetPath), 200
 
 @server.route("/image-classifier/tag-image", methods=['POST'])
 def image_classification():
+    model = _get_model(classification_model, 'image-classification')
     assetPath = request.json['thumbnailPath']
-    return run_engine(classifier, assetPath), 201
+    return run_engine(model, assetPath), 200
 
+@server.route("/sentence-transformer/encode-image", methods=['POST'])
+def clip_encode_image():
+    model = _get_model(clip_image_model)
+    assetPath = request.json['thumbnailPath']
+    return model.encode(Image.open(assetPath)).tolist(), 200
+
+@server.route("/sentence-transformer/encode-text", methods=['POST'])
+def clip_encode_text():
+    model = _get_model(clip_text_model)
+    text = request.json['text']
+    return model.encode(text).tolist(), 200
 
 def run_engine(engine, path):
     result = []
@@ -55,4 +70,4 @@ def run_engine(engine, path):
 
 
 if __name__ == "__main__":
-    server.run(debug=is_dev, host='0.0.0.0', port=server_port)
+    server.run(debug=is_dev, host=server_host, port=server_port)