#-------------------------------------------------------------------- # DEPENDENCIAS #-------------------------------------------------------------------- import os from io import StringIO import requests import gradio as gr import pandas as pd import numpy as np import openai import tiktoken #import streamlit as st from openai.embeddings_utils import get_embedding, cosine_similarity #from langchain.document_loaders import PyPDFLoader #from langchain.text_splitter import CharacterTextSplitter #from PyPDF2 import PdfReader, PdfFileReader from langchain.embeddings import OpenAIEmbeddings, HuggingFaceInstructEmbeddings from langchain.vectorstores import FAISS from langchain.chat_models import ChatOpenAI from langchain.memory import ConversationBufferMemory from langchain.chains import ConversationalRetrievalChain from langchain.llms import OpenAI, HuggingFaceHub from langchain.chains.question_answering import load_qa_chain #from htmlTemplates import css, bot_template, user_template import json import ast #from langchain.schema.vectorstore import Document from langchain.schema import Document #import fitz # PyMuPDF #import pytesseract #from PIL import Image #from io import BytesIO #import cv2 import gspread from oauth2client.service_account import ServiceAccountCredentials from datetime import datetime #-------------------------------------------------------------------- # LLAVES #-------------------------------------------------------------------- openai.api_key = os.getenv("OPENAI_API_KEY") api_key = os.getenv("OPENAI_API_KEY") token = os.getenv("token") headers = { 'Authorization': f'token {token}', 'Accept': 'application/vnd.github.v3.raw' } # Establece las credenciales y la API credentials = os.getenv( "credentials" ) credentials = json.loads( credentials ) gc = gspread.service_account_from_dict( credentials ) Google_URL = os.getenv( "Google_Sheet" ) #-------------------------------------------------------------------- # CARGAR CSV EMBEDDINGS #-------------------------------------------------------------------- # url_tomos_conf_DPR = os.getenv("url_tomos_conf_DPR") response_tomos_conf_DPR = requests.get( url_tomos_conf_DPR, headers = headers ) csv_content_tomos_conf_DPR = response_tomos_conf_DPR.text tomos_conf_DPR = pd.read_csv(StringIO( csv_content_tomos_conf_DPR )) # url_tomos_conf_cita = os.getenv("url_tomos_conf_cita") response_tomos_conf_cita = requests.get( url_tomos_conf_cita, headers = headers ) csv_content_tomos_conf_cita = response_tomos_conf_cita.text tomos_conf_cita = pd.read_csv(StringIO( csv_content_tomos_conf_cita )) # url_df_tomos_1a28_01 = os.getenv("url_df_tomos_1a28_01") response_df_tomos_1a28_01 = requests.get( url_df_tomos_1a28_01, headers = headers ) csv_content_df_tomos_1a28_01 = response_df_tomos_1a28_01.text df_tomos_1a28_01 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_01 )) # url_df_tomos_1a28_02 = os.getenv("url_df_tomos_1a28_02") response_df_tomos_1a28_02 = requests.get( url_df_tomos_1a28_02, headers = headers ) csv_content_df_tomos_1a28_02 = response_df_tomos_1a28_02.text df_tomos_1a28_02 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_02 )) # url_df_tomos_1a28_03 = os.getenv("url_df_tomos_1a28_03") response_df_tomos_1a28_03 = requests.get( url_df_tomos_1a28_03, headers = headers ) csv_content_df_tomos_1a28_03 = response_df_tomos_1a28_03.text df_tomos_1a28_03 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_03 )) # url_df_tomos_1a28_04 = os.getenv("url_df_tomos_1a28_04") response_df_tomos_1a28_04 = requests.get( url_df_tomos_1a28_04, headers = headers ) csv_content_df_tomos_1a28_04 = response_df_tomos_1a28_04.text df_tomos_1a28_04 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_04 )) # url_df_tomos_1a28_05 = os.getenv("url_df_tomos_1a28_05") response_df_tomos_1a28_05 = requests.get( url_df_tomos_1a28_05, headers = headers ) csv_content_df_tomos_1a28_05 = response_df_tomos_1a28_05.text df_tomos_1a28_05 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_05 )) # url_df_tomos_1a28_06 = os.getenv("url_df_tomos_1a28_06") response_df_tomos_1a28_06 = requests.get( url_df_tomos_1a28_06, headers = headers ) csv_content_df_tomos_1a28_06 = response_df_tomos_1a28_06.text df_tomos_1a28_06 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_06 )) # url_df_tomos_1a28_07 = os.getenv("url_df_tomos_1a28_07") response_df_tomos_1a28_07 = requests.get( url_df_tomos_1a28_07, headers = headers ) csv_content_df_tomos_1a28_07 = response_df_tomos_1a28_07.text df_tomos_1a28_07 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_07 )) # url_df_tomos_1a28_08 = os.getenv("url_df_tomos_1a28_08") response_df_tomos_1a28_08 = requests.get( url_df_tomos_1a28_08, headers = headers ) csv_content_df_tomos_1a28_08 = response_df_tomos_1a28_08.text df_tomos_1a28_08 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_08 )) # url_df_tomos_1a28_09 = os.getenv("url_df_tomos_1a28_09") response_df_tomos_1a28_09 = requests.get( url_df_tomos_1a28_09, headers = headers ) csv_content_df_tomos_1a28_09 = response_df_tomos_1a28_09.text df_tomos_1a28_09 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_09 )) # df_tomos_1a28 = pd.concat([df_tomos_1a28_01, df_tomos_1a28_02], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_03], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_04], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_05], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_06], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_07], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_08], ignore_index = True) df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_09], ignore_index = True) # url_tercer_req = os.getenv("url_tercer_req") response_tercer_req = requests.get( url_tercer_req, headers = headers ) csv_content_tercer_req = response_tercer_req.text tercer_req = pd.read_csv(StringIO( csv_content_tercer_req )) # url_seg_req = os.getenv("url_seg_req") response_seg_req = requests.get( url_seg_req, headers = headers ) csv_content_seg_req = response_seg_req.text seg_req = pd.read_csv(StringIO( csv_content_seg_req )) # url_primer_req = os.getenv("url_primer_req") response_primer_req = requests.get( url_primer_req, headers = headers ) csv_content_primer_req = response_primer_req.text primer_req = pd.read_csv(StringIO( csv_content_primer_req )) # url_primer1_req = os.getenv("url_primer1_req") response_primer1_req = requests.get( url_primer1_req, headers = headers ) csv_content_primer1_req = response_primer1_req.text primer1_req = pd.read_csv(StringIO( csv_content_primer1_req )) primer1_req["Folder"] = "I. PRIMER REQUERIMIENTO (139)/2. Desahogo Reiteracion 1 139" # url_primer2_req = os.getenv("url_primer2_req") response_primer2_req = requests.get( url_primer2_req, headers = headers ) csv_content_primer2_req = response_primer2_req.text primer2_req = pd.read_csv(StringIO( csv_content_primer2_req )) primer2_req["Folder"] = "I. PRIMER REQUERIMIENTO (139)/1. Desahogo RFI 139" #--------------------------------------------------------------------------------------------------------------- # UUUUPS LA COLUMNA EMBEDDINGS NO LA RECONOCE COSINESIMILARITY.. [tomos_conf_DPR, tomos_conf_cita] #--------------------------------------------------------------------------------------------------------------- def clean_and_parse_embedding(embedding_str): # Extract the part between square brackets embedding_str = embedding_str.split('[')[-1].split(']')[0] # Now, you should have a clean string representation of the list embedding_list = ast.literal_eval(embedding_str) return [float(val) for val in embedding_list] tomos_conf_DPR['Embedding'] = tomos_conf_DPR['Embedding'].apply(clean_and_parse_embedding) tomos_conf_cita['Embedding'] = tomos_conf_cita['Embedding'].apply(clean_and_parse_embedding) tercer_req['Embedding'] = tercer_req['Embedding'].apply(clean_and_parse_embedding) seg_req['Embedding'] = seg_req['Embedding'].apply(clean_and_parse_embedding) primer_req['Embedding'] = primer_req['Embedding'].apply(clean_and_parse_embedding) primer1_req['Embedding'] = primer1_req['Embedding'].apply(clean_and_parse_embedding) primer2_req['Embedding'] = primer2_req['Embedding'].apply(clean_and_parse_embedding) #--------------------------------------------------------------------------------------------------------------- # UUUUPS LA COLUMNA EMBEDDINGS NO LA RECONOCE COSINESIMILARITY.. [df_tomos_1a28] #--------------------------------------------------------------------------------------------------------------- def parse_embedding(embedding_str): embedding_list = ast.literal_eval(embedding_str) return [float(val) for val in embedding_list] df_tomos_1a28['Embedding'] = df_tomos_1a28['Embedding'].apply(parse_embedding) #--------------------------------------------------------------------------------------------------------------- # LISTA DE DF #--------------------------------------------------------------------------------------------------------------- list_of_dfs = [tomos_conf_DPR, tomos_conf_cita, df_tomos_1a28, tercer_req, seg_req, primer_req, primer1_req, primer2_req] #-------------------------------------------------------------------- # HACEMOS UNA PREGUNTA Y RANKEA CHUNKS #-------------------------------------------------------------------- def buscar(busqueda, lista_de_datos): resultados = [] # Create an empty list to store individual DataFrame results busqueda_embed = get_embedding(busqueda, engine="text-embedding-ada-002") for datos in lista_de_datos: datos["similitud"] = datos['Embedding'].apply(lambda x: cosine_similarity(x, busqueda_embed)) datos = datos.sort_values("similitud", ascending=False) resultados.append(datos[['PDFName', 'PageNumber', 'similitud', "PageText", "Folder"]]) # Concatenate all individual DataFrames into a single DataFrame combined_result = pd.concat(resultados).sort_values("similitud", ascending=False).head(20) return combined_result #-------------------------------------------------------------------- # rank for ai #-------------------------------------------------------------------- def buscar_ai(busqueda, lista_de_datos): resultados = [] # Create an empty list to store individual DataFrame results busqueda_embed = get_embedding(busqueda, engine="text-embedding-ada-002") for datos in lista_de_datos: datos["similitud"] = datos['Embedding'].apply(lambda x: cosine_similarity(x, busqueda_embed)) datos = datos.sort_values("similitud", ascending=False) resultados.append(datos[['PDFName', 'PageNumber', 'similitud', "PageText", "Folder"]]) # Concatenate all individual DataFrames into a single DataFrame combined_result = pd.concat(resultados).sort_values("similitud", ascending=False).head(10) return combined_result #-------------------------------------------------------------------- # saque n extraactos de "" #-------------------------------------------------------------------- def count_text_extracted(pregunta): df = buscar(pregunta, list_of_dfs) pdf_counts = df.groupby(['Folder', 'PDFName'])['PageNumber'].count().reset_index() output_string = "" for idx, row in pdf_counts.iterrows(): folder_name = row['Folder'] pdf_name = row['PDFName'] count = row['PageNumber'] page_numbers = df[(df['PDFName'] == pdf_name) & (df['Folder'] == folder_name)]['PageNumber'].tolist() page_numbers_str = ', '.join(map(str, page_numbers)) output_string += f"Usé el archivo '{pdf_name}' del folder '{folder_name}' {count} (vez/veces) al extraer el texto de las páginas {page_numbers_str}.\n\n" return output_string #-------------------------------------------------------------------- # file: texto #-------------------------------------------------------------------- def print_pdf_info(pregunta): df = buscar(pregunta, list_of_dfs) output_string = "" # Initialize an empty string to accumulate the output for _, row in df.iterrows(): pdf_name = row['PDFName'] page_number = row['PageNumber'] page_text = row['PageText'] # Split page_text into lines and add a tab to each line indented_page_text = '\n'.join(['\t' + line for line in page_text.split('\n')]) # Append the formatted output to the output string output_string += f'De "{pdf_name}":\n \tPágina {page_number}:\n\t {indented_page_text}\n' return output_string #-------------------------------------------------------------------- # vector -> document #------------------------------------------------------------------- def vector_document(dataframe): string_vectors = dataframe["PageText"] documents = [Document(page_content=content, metadata={'id': i}) for i, content in enumerate(string_vectors)] return documents #-------------------------------------------------------------------- # AI QUESTION #------------------------------------------------------------------- def info_pdf(pregunta): df = buscar(pregunta, list_of_dfs) output_list = [] # Initialize an empty list to store the output for _, row in df.iterrows(): pdf_name = row['PDFName'] page_number = row['PageNumber'] page_text = row['PageText'] # Split page_text into lines and add a tab to each line indented_page_text = '\n'.join(['\t' + line for line in page_text.split('\n')]) # Append the formatted output to the output list output_list.append(f'De "{pdf_name}": Página {page_number}: {indented_page_text}') return output_list def get_completion_from_messages( messages, model = "gpt-3.5-turbo-16k", temperature = 0, max_tokens = 4500 ): ##Check max_tokens response = openai.ChatCompletion.create( model = model, messages = messages, temperature = temperature, max_tokens = max_tokens, ) return response.choices[0].message["content"] def get_topic( user_message ): # delimiter = "####" system_message = f""" Eres un abogado que trabaja en temas de competencia económica e investiga casos en México. Siempre intenarás responder en el mayor número posible de palabras. Las consultas o preguntas se delimitarán con los caracteres {delimiter} """ # messages = [ {'role':'system', 'content': system_message}, {'role':'user', 'content': f"{delimiter}{user_message}{delimiter}"}, ] return get_completion_from_messages( messages ) def get_respuesta( user_message, informacion): # delimiter = "####" system_message = f""" Eres un abogado que trabaja en temas de competencia económica e investiga casos en México. Siempre intenarás responder en el mayor número posible de palabras. Las consultas o preguntas se delimitarán con los caracteres {delimiter} """ # messages = [ {'role':'system', 'content': system_message}, {'role':'user', 'content': f""" {delimiter} Estás intentando recopilar información relevante para tu caso. Usa exclusivamente la información contenida en la siguiente lista: {informacion} para responder sin límite de palabras lo siguiente: {user_message} Responde de forma detallada. {delimiter} """}, ] # return get_completion_from_messages(messages) def update_records( user_message ): # sht = gc.open_by_url(Google_URL) # sht.worksheet("Hoja 2").get_all_records() # sht.worksheet("Hoja 2").update_cell( len( sht.worksheet("Hoja 2").get_all_records()[:] ) + 2 , 1 , datetime.now().strftime("%m/%d/%Y, %H:%M:%S") ) # sht.worksheet("Hoja 2").update_cell( len( sht.worksheet("Hoja 2").get_all_records()[:] ) + 1 , 2 , user_message ) def chat(user_message_1): # norma_y_tema_response_1 = get_topic(user_message_1) norma_y_tema_response_1 += 'Todos' uno = buscar_ai(user_message_1, list_of_dfs) lista_info = uno['PageText'].tolist() # # Save Question and date time update_records( user_message_1 ) # return get_respuesta(user_message_1, lista_info) # Modify your existing code with gr.Blocks() as demo: txt = gr.Textbox(label="Texto", lines=2) btn = gr.Button(value="Listo") txt_2 = gr.Textbox(value="", label="Donde (top 20):") txt_3 = gr.Textbox(value="", label="Extractos (top 20):") txt_1 = gr.Textbox(value="", label="Respuesta IA:") btn.click(chat, inputs=[txt], outputs=[txt_1]) btn.click(count_text_extracted, inputs=[txt], outputs=[txt_2]) btn.click(print_pdf_info, inputs=[txt], outputs=[txt_3]) if __name__ == "__main__": demo.launch(share=True)