File size: 7,431 Bytes
1863152 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 |
import textwrap
import numpy as np
import pandas as pd
from transformers import BlipProcessor, BlipForConditionalGeneration, DetrImageProcessor, DetrForObjectDetection
from PyPDF2 import PdfReader
import google.generativeai as genai
import google.ai.generativelanguage as glm
from PIL import Image
import torch
from IPython.display import Markdown
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import FAISS
class ImageProcessor:
def __init__(self, image_path):
self.image_path = image_path
def get_caption(self, image_path):
# Implement image captioning logic here
"""
Generates a short caption for the provided image.
Args:
image_path (str): The path to the image file.
Returns:
str: A string representing the caption for the image.
"""
image = Image.open(image_path).convert('RGB')
model_name = "Salesforce/blip-image-captioning-large"
device = "cpu" # cuda
processor = BlipProcessor.from_pretrained(model_name)
model = BlipForConditionalGeneration.from_pretrained(model_name).to(device)
inputs = processor(image, return_tensors='pt').to(device)
output = model.generate(**inputs, max_new_tokens=20)
caption = processor.decode(output[0], skip_special_tokens=True)
return caption
def detect_objects(self, image_path):
# Implement object detection logic here
"""
Detects objects in the provided image.
Args:
image_path (str): The path to the image file.
Returns:
str: A string with all the detected objects. Each object as '[x1, x2, y1, y2, class_name, confindence_score]'.
"""
image = Image.open(image_path).convert('RGB')
processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50")
model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50")
inputs = processor(images=image, return_tensors="pt")
outputs = model(**inputs)
# convert outputs (bounding boxes and class logits) to COCO API
# let's only keep detections with score > 0.9
target_sizes = torch.tensor([image.size[::-1]])
results = processor.post_process_object_detection(outputs, target_sizes=target_sizes, threshold=0.9)[0]
detections = ""
for score, label, box in zip(results["scores"], results["labels"], results["boxes"]):
detections += '[{}, {}, {}, {}]'.format(int(box[0]), int(box[1]), int(box[2]), int(box[3]))
detections += ' {}'.format(model.config.id2label[int(label)])
detections += ' {}\n'.format(float(score))
return detections
def make_prompt(self, query, image_captions, objects_detections):
# Implement prompt creation logic here
escaped_captions = image_captions.replace("'", "").replace('"', "").replace("\n", " ")
escaped_objects = objects_detections.replace("'", "").replace('"', "").replace("\n", " ")
prompt = textwrap.dedent("""You are a helpful and informative bot that answers questions using text from the image captions and objects detected included below. \
Be sure to respond in a complete sentence, being comprehensive, including all relevant background information. \
However, you are talking to a non-technical audience, so be sure to break down complicated concepts and \
strike a friendly and conversational tone. \
If the image captions or objects detected are irrelevant to the answer, you may ignore them.
QUESTION: '{query}'
IMAGE CAPTIONS: '{image_captions}'
OBJECTS DETECTED: '{objects_detected}'
ANSWER:
""").format(query=query, image_captions=escaped_captions, objects_detected=escaped_objects)
return prompt
def generate_answer(self, prompt):
# Implement answer generation logic here
model = genai.GenerativeModel('gemini-pro')
answer = model.generate_content(prompt)
return answer.text
class PDFProcessor:
def __init__(self, pdf_path):
self.pdf_path = pdf_path
def create_embedding_df(self, pdf_path):
# Implement PDF content vector store creation logic here
# Provide the path of the PDF file
pdfreader = PdfReader(pdf_path)
# Read text from PDF and divide it into smaller chunks
documents = []
for i, page in enumerate(pdfreader.pages):
content = page.extract_text()
if content:
# Create a document for each page
document = {
"Title": f"Page {i+1}", # Use the page number as the title
"Text": content
}
documents.append(document)
# Create a DataFrame from the documents
df = pd.DataFrame(documents)
# Define the model
model = 'models/embedding-001'
# Define a function to generate embeddings
def embed_fn(title, text):
return genai.embed_content(
model=model,
content=text,
task_type="retrieval_document",
title=title
)["embedding"]
# Generate embeddings for each document and store them in the DataFrame
df['Embeddings'] = df.apply(lambda row: embed_fn(row['Title'], row['Text']), axis=1)
return df
def find_best_passage(self, query, dataframe):
# Implement logic to find the best passage based on query
"""
Compute the distances between the query and each document in the dataframe
using the dot product.
"""
model = 'models/embedding-001'
query_embedding = genai.embed_content(model=model,
content=query,
task_type="retrieval_query")
dot_products = np.dot(np.stack(dataframe['Embeddings']), query_embedding["embedding"])
idx = np.argmax(dot_products)
# Return text from index with max value
return dataframe.iloc[idx]['Text']
def make_prompt(self, query, relevant_passage):
# Implement prompt creation logic for PDF processing
escaped = relevant_passage.replace("'", "").replace('"', "").replace("\n", " ")
prompt = textwrap.dedent("""You are a helpful and informative bot that answers questions using text from the reference passage included below. \
Be sure to respond in a complete sentence, being comprehensive, including all relevant background information. \
However, you are talking to a non-technical audience, so be sure to break down complicated concepts and \
strike a friendly and converstional tone. \
If the passage is irrelevant to the answer, you may ignore it.
QUESTION: '{query}'
PASSAGE: '{relevant_passage}'
ANSWER:
""").format(query=query, relevant_passage=escaped)
return prompt
def generate_answer(self, prompt):
# Implement answer generation logic for PDF processing
model = genai.GenerativeModel('gemini-pro')
answer = model.generate_content(prompt)
return answer.text
|