Update app.py

app.py

@@ -269,10 +269,61 @@ def predict_batch(images, urls):
         
     return batch_results
 
+# Fonction de prédiction avec texte
+def predict_with_text(image, url, text_prompt):
+    if url:
+        response = requests.get(url)
+        image = Image.open(BytesIO(response.content))
+    
+    processed_image = preprocess_val(image).unsqueeze(0).to(device)
+    
+    # Encoder l'image
+    with torch.no_grad(), torch.amp.autocast(device_type=device):
+        image_features = model.encode_image(processed_image)
+        image_features /= image_features.norm(dim=-1, keepdim=True)
+        
+        # Encoder le texte fourni par l'utilisateur
+        user_text = tokenizer([text_prompt]).to(device)
+        user_text_features = model.encode_text(user_text)
+        user_text_features /= user_text_features.norm(dim=-1, keepdim=True)
+        
+        # Combiner les caractéristiques de l'image et du texte (moyenne pondérée)
+        combined_features = 0.7 * image_features + 0.3 * user_text_features
+        combined_features /= combined_features.norm(dim=-1, keepdim=True)
+        
+        # Calculer les probabilités avec les caractéristiques combinées
+        text_probs = (100 * combined_features @ text_features.T).softmax(dim=-1)
+        
+        sorted_confidences = sorted(
+            {items[i]: float(text_probs[0, i]) for i in range(len(items))}.items(), 
+            key=lambda x: x[1], 
+            reverse=True
+        )
+        
+        # Inclure les IDs de catégorie dans la réponse
+        top_10_categories = [
+            {
+                "category_name": category["name"],
+                "id": category["id"],
+                "confidence": confidence
+            }
+            for category_name, confidence in sorted_confidences[:10]
+            for category in categories if category["name"] == category_name
+        ]
+        
+    return image, top_10_categories
+
+# Fonction de prédiction combinée qui choisit la méthode appropriée
+def predict_combined(image, url, text_prompt=""):
+    if text_prompt and text_prompt.strip():
+        return predict_with_text(image, url, text_prompt)
+    else:
+        return predict(image, url)
+
 # Clear function
 def clear_fields():
-    # return None, "", None, ""
-    return None, ""
+    return None, "", "", None, ""
+
 # Gradio interface
 title = "Fashion Item Classifier with Marqo-FashionSigLIP"
 description = "Upload an image or provide a URL of a fashion item to classify it using [Marqo-FashionSigLIP](https://huggingface.co/Marqo/marqo-fashionSigLIP)!"
@@ -292,19 +343,18 @@ with gr.Blocks() as demo:
         with gr.Column(scale=2):
             input_image = gr.Image(type="pil", label="Upload Fashion Item Image", height=312)
             input_url = gr.Textbox(label="Or provide an image URL")
-            # input_images = gr.Image(type="pil", label="Upload Fashion Item Images", height=312)
-            # input_urls = gr.Textbox(label="Or provide image URLs (comma-separated)", lines=2)
+            input_text = gr.Textbox(label="Ajouter une description textuelle (optionnel)", placeholder="Ex: Robe d'été fleurie pour femme")
+            input_images = gr.Image(type="pil", label="Upload Fashion Item Images", height=312)
+            input_urls = gr.Textbox(label="Or provide image URLs (comma-separated)", lines=2)
             with gr.Row():
-                predict_button = gr.Button("Classify")
-                # predict_batch_button = gr.Button("Classify Batch")
-                clear_button = gr.Button("Clear")
-            gr.Markdown("Or click on one of the images below to classify it:")
+                predict_button = gr.Button("Classifier")
+                clear_button = gr.Button("Effacer")
+            gr.Markdown("Ou cliquez sur l'une des images ci-dessous pour la classifier:")
             gr.Examples(examples=examples, inputs=input_image)
             output_label = gr.JSON(label="Top Categories")
-            # output_batch_label = gr.JSON(label="Top Categories for Each Image")
-            predict_button.click(predict, inputs=[input_image, input_url], outputs=[input_image, output_label])
-            # predict_batch_button.click(predict_batch, inputs=[input_images, input_urls], outputs=output_batch_label)
-            # clear_button.click(clear_fields, outputs=[input_image, input_url, input_images, input_urls])
+            output_batch_label = gr.JSON(label="Top Categories for Each Image")
+            predict_button.click(predict_combined, inputs=[input_image, input_url, input_text], outputs=[input_image, output_label])
+            clear_button.click(clear_fields, outputs=[input_image, input_url, input_text, input_images, input_urls])
         
 # Launch the interface
 demo.launch()