main

Readme.md

@@ -0,0 +1,26 @@
+# PaliGemma Image Captioning Gradio App
+
+## Deployment Instructions
+
+1. Create a new Hugging Face Space
+2. Choose Python as the SDK
+3. Select 16GB CPU hardware
+4. Upload the following files:
+   - `app.py`
+   - `requirements.txt`
+
+### HuggingFace Access Token
+
+1. Go to HuggingFace settings
+2. Create a new access token with "Read" permissions
+3. Add the token as a secret named `HF_TOKEN` in your Space settings
+
+### Features
+- Multi-language image captioning
+- Upload custom images
+- Example images included
+- Supports English, Spanish, French, German captions
+
+## Model Details
+- Model: google/paligemma-3b-mix-224
+- Task: Multilingual Image Captioning

app.py

@@ -1,74 +1,85 @@
 import gradio as gr
-from PIL import Image 
-from transformers import AutoModelForCausalLM 
-from transformers import AutoProcessor 
-from transformers import TextIteratorStreamer
-from threading import Thread
+from transformers import AutoProcessor, PaliGemmaForConditionalGeneration
+from PIL import Image
 import torch
-import spaces
+import requests
 
-model_id = "microsoft/Phi-3-vision-128k-instruct"
-model = AutoModelForCausalLM.from_pretrained(
-    model_id, 
-    device_map="cpu", 
-    trust_remote_code=True, 
-    torch_dtype=torch.float32,
-    _attn_implementation="eager"
-)
-processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+# Load the model and processor
+model_id = "google/paligemma-3b-mix-224"
+model = PaliGemmaForConditionalGeneration.from_pretrained(model_id, token=True).eval()
+processor = AutoProcessor.from_pretrained(model_id, token=True)
 
-@spaces.CPU
-def bot_streaming(message, history):
-    try:
-        image = (message["files"][-1]["path"] if isinstance(message["files"][-1], dict) else message["files"][-1]) if message["files"] else None
-        
-        if not image:
-            raise ValueError("No image uploaded")
-
-        conversation = []
-        for user, assistant in history:
-            conversation.extend([
-                {"role": "user", "content": user}, 
-                {"role": "assistant", "content": assistant or ""}
-            ])
-        
-        conversation.append({"role": "user", "content": f"<|image_1|>\n{message['text']}"})
-        
-        prompt = processor.tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
-        image = Image.open(image)
-        inputs = processor(prompt, image, return_tensors="pt")
+# Supported languages and example prompts
+LANGUAGES = {
+    "English": "caption en",
+    "Spanish": "caption es", 
+    "French": "caption fr",
+    "German": "caption de"
+}
 
-        streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
-        generation_kwargs = dict(
-            inputs, 
-            streamer=streamer, 
-            max_new_tokens=256,
-            do_sample=False, 
-            temperature=0.1, 
-            eos_token_id=processor.tokenizer.eos_token_id
-        )
+def generate_caption(image, language, max_tokens=100):
+    """Generate image caption in specified language"""
+    if image is None:
+        return "Please upload an image."
+    
+    prompt = LANGUAGES.get(language, "caption en")
+    
+    # Preprocess inputs
+    model_inputs = processor(text=prompt, images=image, return_tensors="pt")
+    input_len = model_inputs["input_ids"].shape[-1]
+    
+    # Generate caption
+    with torch.inference_mode():
+        generation = model.generate(**model_inputs, max_new_tokens=max_tokens, do_sample=False)
+        generation = generation[0][input_len:]
+        decoded = processor.decode(generation, skip_special_tokens=True)
+    
+    return decoded
 
-        thread = Thread(target=model.generate, kwargs=generation_kwargs)
-        thread.start()
+def load_example_image(url):
+    """Load example image from URL"""
+    return Image.open(requests.get(url, stream=True).raw)
 
-        buffer = ""
-        for new_text in streamer:
-            buffer += new_text
-            yield buffer
+# Prepare example images
+EXAMPLE_IMAGES = [
+    load_example_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg"),
+    load_example_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/food.jpg"),
+    load_example_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/city.jpg")
+]
 
-    except Exception as e:
-        yield f"Error: {str(e)}"
-
-demo = gr.Blocks()
-with demo:
-    gr.ChatInterface(
-        fn=bot_streaming,
-        title="Phi3 Vision 128K",
-        description="Multimodal AI Vision Model",
-        examples=[
-            {"text": "Describe this image", "files": ["./example.jpg"]},
-        ]
+# Create Gradio Interface
+with gr.Blocks() as demo:
+    gr.Markdown("# PaliGemma Image Captioning")
+    gr.Markdown("Upload an image and get a caption in your preferred language!")
+    
+    with gr.Row():
+        with gr.Column():
+            input_image = gr.Image(type="pil", label="Upload Image")
+            language_dropdown = gr.Dropdown(
+                list(LANGUAGES.keys()), 
+                value="English", 
+                label="Caption Language"
+            )
+            submit_btn = gr.Button("Generate Caption")
+        
+        with gr.Column():
+            output_text = gr.Textbox(label="Generated Caption")
+    
+    # Connect components
+    submit_btn.click(
+        fn=generate_caption, 
+        inputs=[input_image, language_dropdown], 
+        outputs=output_text
+    )
+    
+    # Add example images
+    gr.Examples(
+        examples=[[img, lang] for img in EXAMPLE_IMAGES for lang in LANGUAGES.keys()],
+        inputs=[input_image, language_dropdown],
+        fn=generate_caption,
+        outputs=output_text
     )
 
-demo.queue()
-demo.launch(debug=True)
+# Launch the app
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -1,6 +1,6 @@
-#flash_attn
-accelerate
-git+https://github.com/huggingface/transformers.git@main
-spaces
-torchvision
-
+torch>=2.0.0
+transformers>=4.37.0
+gradio>=4.0.0
+pillow>=10.0.0
+huggingface_hub
+requests