app.py

import gradio as gr
from transformers import AutoProcessor, PaliGemmaForConditionalGeneration, BitsAndBytesConfig
from peft import PeftModel
import spaces
import torch
from huggingface_hub.hf_api import HfFolder
import os
token = os.getenv('token')
HfFolder.save_token(token)
device = "cuda"
model = PaliGemmaForConditionalGeneration.from_pretrained("triphuong57/paligemma_lora").to("cuda")
processor = AutoProcessor.from_pretrained("google/paligemma-3b-mix-224")

@spaces.GPU(duration=180)
def greet(image, prompt):
    
    
    # model = PaliGemmaForConditionalGeneration.from_pretrained("/folders", torch_dtype=torch.float16, quantization_config=quantization_config).to(device)
    
    
    # # model = PeftModel(base_model, "/folders").to(device)
    model_inputs = processor(text=prompt, images=image, return_tensors="pt")
    input_len = model_inputs["input_ids"].shape[-1]
    with torch.inference_mode():
        generation = model.generate(**model_inputs, max_new_tokens=100, do_sample=False)
        decoded = processor.decode(generation[0][input_len:], skip_special_tokens=True)
    return decoded

title = "Demo BTL nhóm 8"
description = "Made by Nguyễn Quý Đang, Đỗ Minh Nhật, Vũ Vân Long"
demo = gr.Interface(fn=greet, inputs=[gr.Image(label="Upload image", sources=['upload', 'webcam'], type="pil"), gr.Text()], outputs="text", title=title, description=description)
demo.launch(share=True)