handler.py creation

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

handler.py

@@ -0,0 +1,62 @@
+from typing import Dict, Any
+import torch
+from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
+from PIL import Image
+import requests
+from io import BytesIO
+
+# Check for GPU
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+class EndpointHandler:
+    def __init__(self, path: str = "morthens/qwen2-vl-inference"):
+        # Load the processor and model
+        self.processor = AutoProcessor.from_pretrained(path)
+        self.model = Qwen2VLForConditionalGeneration.from_pretrained(
+            path,
+            torch_dtype="auto",
+            device_map="auto"
+        )
+        # Move the model to the appropriate device
+        self.model.to(device)
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        # Extract the input data
+        image_url = data.get("image_url", "")
+        text = data.get("text", "")
+
+        # Load the image from the URL
+        try:
+            response = requests.get(image_url)
+            response.raise_for_status()
+            image = Image.open(BytesIO(response.content))
+        except Exception as e:
+            return {"error": f"Failed to fetch or process image: {str(e)}"}
+
+        # Preprocess the input
+        inputs = self.processor(
+            text=[text],
+            images=[image],
+            padding=True,
+            return_tensors="pt"
+        )
+
+        # Move inputs to the correct device
+        inputs = {key: value.to(device) for key, value in inputs.items()}
+
+        # Perform inference
+        output_ids = self.model.generate(
+            **inputs,
+            max_new_tokens=128
+        )
+
+        # Decode the output
+        output_text = self.processor.batch_decode(
+            output_ids,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=True
+        )[0]
+
+        # Return the raw prediction
+        return {"prediction": output_text}
+

requirements.txt

@@ -0,0 +1,3 @@
+transformers==4.45.0
+optimum
+auto_gptq