Create app.py

app.py

@@ -0,0 +1,172 @@
+import os
+import io
+import base64
+import torch
+import gradio as gr
+import google.generativeai as genai
+from PIL import Image
+from langchain_core.prompts import PromptTemplate
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain.chains.question_answering import load_qa_chain
+from transformers import AutoTokenizer, AutoModelForCausalLM, LlavaNextForConditionalGeneration, LlavaNextProcessor
+from pypdf import PdfReader
+from doctr.io import DocumentFile
+from doctr.models import ocr_predictor
+import chromadb
+from chromadb.utils import embedding_functions
+from chromadb.utils.data_loaders import ImageLoader
+
+# Configure Gemini API
+genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+
+# Load Mistral model
+model_path = "nvidia/Mistral-NeMo-Minitron-8B-Base"
+mistral_tokenizer = AutoTokenizer.from_pretrained(model_path)
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+dtype = torch.bfloat16
+mistral_model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype=dtype, device_map=device)
+
+# Load OCR model
+ocr_model = ocr_predictor(
+    "db_resnet50",
+    "crnn_mobilenet_v3_large",
+    pretrained=True,
+    assume_straight_pages=True,
+)
+
+# Load Llava model for image description
+if torch.cuda.is_available():
+    processor = LlavaNextProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
+    vision_model = LlavaNextForConditionalGeneration.from_pretrained(
+        "llava-hf/llava-v1.6-mistral-7b-hf",
+        torch_dtype=torch.float16,
+        low_cpu_mem_usage=True,
+        load_in_4bit=True,
+    )
+
+def extract_images(pdf_path):
+    images = []
+    pdf_document = DocumentFile.from_pdf(pdf_path)
+    for page in range(len(pdf_document)):
+        page_images = pdf_document.get_page_images(page)
+        images.extend(page_images)
+    return images
+
+def get_image_description(image):
+    torch.cuda.empty_cache()
+    prompt = "[INST] <image>\nDescribe the image in a sentence [/INST]"
+    inputs = processor(prompt, image, return_tensors="pt").to(device)
+    output = vision_model.generate(**inputs, max_new_tokens=100)
+    return processor.decode(output[0], skip_special_tokens=True)
+
+def extract_text_with_ocr(pdf_path):
+    pdf_doc = DocumentFile.from_pdf(pdf_path)
+    result = ocr_model(pdf_doc)
+    return "\n".join([block.text for page in result.pages for block in page.blocks])
+
+def get_vectordb(text, images):
+    client = chromadb.EphemeralClient()
+    loader = ImageLoader()
+    sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
+        model_name="multi-qa-mpnet-base-dot-v1"
+    )
+    
+    text_collection = client.get_or_create_collection(
+        name="text_db",
+        embedding_function=sentence_transformer_ef,
+    )
+    image_collection = client.get_or_create_collection(
+        name="image_db",
+        embedding_function=sentence_transformer_ef,
+        data_loader=loader,
+    )
+    
+    # Add text to vector database
+    text_collection.add(
+        ids=["1"],
+        documents=[text],
+    )
+    
+    # Add images to vector database
+    for i, image in enumerate(images):
+        img_description = get_image_description(image)
+        image_collection.add(
+            ids=[f"img_{i}"],
+            documents=[img_description],
+            metadatas=[{"image": image_to_bytes(image)}],
+        )
+    
+    return client
+
+def image_to_bytes(image):
+    buffered = io.BytesIO()
+    image.save(buffered, format="PNG")
+    return base64.b64encode(buffered.getvalue()).decode()
+
+def process_pdf(file_path):
+    # Extract text using OCR
+    text = extract_text_with_ocr(file_path)
+    
+    # Extract images
+    images = extract_images(file_path)
+    
+    # Create vector database
+    vectordb = get_vectordb(text, images)
+    
+    return vectordb, text, images
+
+def answer_question(vectordb, question):
+    model = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3)
+    prompt_template = """Answer the question as precisely as possible using the provided context. If the answer is
+                          not contained in the context, say "answer not available in context" \n\n
+                          Context: \n {context}?\n
+                          Question: \n {question} \n
+                          Answer:
+                        """
+    prompt = PromptTemplate(template=prompt_template, input_variables=["context", "question"])
+    
+    # Query text database
+    text_collection = vectordb.get_collection("text_db")
+    text_results = text_collection.query(query_texts=[question], n_results=1)
+    
+    # Query image database
+    image_collection = vectordb.get_collection("image_db")
+    image_results = image_collection.query(query_texts=[question], n_results=1)
+    
+    context = f"Text: {text_results['documents'][0][0]}\nImage: {image_results['documents'][0][0]}"
+    
+    stuff_chain = load_qa_chain(model, chain_type="stuff", prompt=prompt)
+    stuff_answer = stuff_chain({"input_documents": [context], "question": question}, return_only_outputs=True)
+    gemini_answer = stuff_answer['output_text']
+    
+    # Use Mistral model for additional text generation
+    mistral_prompt = f"Based on this answer: {gemini_answer}\nGenerate a follow-up question:"
+    mistral_inputs = mistral_tokenizer.encode(mistral_prompt, return_tensors='pt').to(device)
+    with torch.no_grad():
+        mistral_outputs = mistral_model.generate(mistral_inputs, max_length=50)
+    mistral_output = mistral_tokenizer.decode(mistral_outputs[0], skip_special_tokens=True)
+    
+    combined_output = f"Gemini Answer: {gemini_answer}\n\nMistral Follow-up: {mistral_output}"
+    return combined_output
+
+def pdf_qa(file, question):
+    if file is None:
+        return "Please upload a PDF file first."
+    
+    vectordb, text, images = process_pdf(file.name)
+    return answer_question(vectordb, question)
+
+# Define Gradio Interface
+input_file = gr.File(label="Upload PDF File")
+input_question = gr.Textbox(label="Ask about the document")
+output_text = gr.Textbox(label="Answer - Combined Gemini and Mistral")
+
+# Create Gradio Interface
+gr.Interface(
+    fn=pdf_qa,
+    inputs=[input_file, input_question],
+    outputs=output_text,
+    title="Advanced PDF Analysis and QA System",
+    description="Upload a PDF file and ask questions about the content, including text and images."
+).launch()