File size: 6,762 Bytes
3dc0491 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 |
import streamlit as st
import pandas as pd
import plotly.express as px
from datasets import load_dataset
from pandasai import Agent
from langchain_community.embeddings.openai import OpenAIEmbeddings
from langchain_community.vectorstores import FAISS
from langchain_openai import ChatOpenAI
from langchain.chains import RetrievalQA
from langchain.schema import Document
import os
import logging
# Configure logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
# Fetch API keys from environment variables
api_key = os.getenv("OPENAI_API_KEY")
pandasai_api_key = os.getenv("PANDASAI_API_KEY")
# Check for missing keys and raise specific errors
missing_keys = []
if not api_key:
missing_keys.append("OPENAI_API_KEY")
if not pandasai_api_key:
missing_keys.append("PANDASAI_API_KEY")
if missing_keys:
missing_keys_str = ", ".join(missing_keys)
raise EnvironmentError(
f"The following API key(s) are missing: {missing_keys_str}. Please set them in the environment."
)
# Title of the app
st.title("Data Analyzer")
# Function to load datasets into session
def load_dataset_into_session():
input_option = st.radio(
"Select Dataset Input:",
["Use Repo Directory Dataset", "Use Hugging Face Dataset", "Upload CSV File"],
)
# Option 1: Load dataset from the repo directory
if input_option == "Use Repo Directory Dataset":
file_path = "./source/test.csv"
if st.button("Load Repo Dataset"):
try:
st.session_state.df = pd.read_csv(file_path)
st.success(f"File loaded successfully from '{file_path}'!")
st.dataframe(st.session_state.df.head(10))
except Exception as e:
st.error(f"Error loading dataset from the repo directory: {e}")
logger.error(f"Error loading dataset from repo directory: {e}")
# Option 2: Load dataset from Hugging Face
elif input_option == "Use Hugging Face Dataset":
dataset_name = st.text_input(
"Enter Hugging Face Dataset Name:", value="HUPD/hupd"
)
if st.button("Load Hugging Face Dataset"):
try:
dataset = load_dataset(dataset_name, split="train", trust_remote_code=True)
# Convert Hugging Face dataset to Pandas DataFrame
if hasattr(dataset, "to_pandas"):
st.session_state.df = dataset.to_pandas()
else:
st.session_state.df = pd.DataFrame(dataset)
st.success(f"Hugging Face Dataset '{dataset_name}' loaded successfully!")
st.dataframe(st.session_state.df.head(10))
except Exception as e:
st.error(f"Error loading Hugging Face dataset: {e}")
logger.error(f"Error loading Hugging Face dataset: {e}")
# Option 3: Upload CSV File
elif input_option == "Upload CSV File":
uploaded_file = st.file_uploader("Upload a CSV File:", type=["csv"])
if uploaded_file:
try:
st.session_state.df = pd.read_csv(uploaded_file)
st.success("File uploaded successfully!")
st.dataframe(st.session_state.df.head(10))
except Exception as e:
st.error(f"Error reading uploaded file: {e}")
logger.error(f"Error reading uploaded file: {e}")
# Ensure session state for the DataFrame
if "df" not in st.session_state:
st.session_state.df = None
# Load dataset into session
load_dataset_into_session()
# Check if a dataset is loaded
if st.session_state.df is not None:
df = st.session_state.df
try:
# Initialize PandasAI Agent
agent = Agent(df)
# Convert DataFrame to documents for RAG
documents = [
Document(
page_content=", ".join(
[f"{col}: {row[col]}" for col in df.columns if pd.notnull(row[col])]
),
metadata={"index": index},
)
for index, row in df.iterrows()
]
# Set up RAG
embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_documents(documents, embeddings)
retriever = vectorstore.as_retriever()
qa_chain = RetrievalQA.from_chain_type(
llm=ChatOpenAI(),
chain_type="stuff",
retriever=retriever,
)
# Create tabs
tab1, tab2, tab3 = st.tabs(
["PandasAI Analysis", "RAG Q&A", "Data Visualization"]
)
# Tab 1: PandasAI Analysis
with tab1:
st.header("PandasAI Analysis")
pandas_question = st.text_input("Ask a question about the data (PandasAI):")
if pandas_question:
try:
result = agent.chat(pandas_question)
st.write("PandasAI Answer:", result)
except Exception as e:
st.error(f"Error during PandasAI Analysis: {e}")
# Tab 2: RAG Q&A
with tab2:
st.header("RAG Q&A")
rag_question = st.text_input("Ask a question about the data (RAG):")
if rag_question:
try:
result = qa_chain.run(rag_question)
st.write("RAG Answer:", result)
except Exception as e:
st.error(f"Error during RAG Q&A: {e}")
# Tab 3: Data Visualization
with tab3:
st.header("Data Visualization")
viz_question = st.text_input(
"What kind of graph would you like to create? (e.g., 'Show a scatter plot of salary vs experience')"
)
if viz_question:
try:
result = agent.chat(viz_question)
# Extract Python code for visualization
import re
code_pattern = r"```python\n(.*?)\n```"
code_match = re.search(code_pattern, result, re.DOTALL)
if code_match:
viz_code = code_match.group(1)
# Replace matplotlib (plt) code with Plotly (px)
viz_code = viz_code.replace("plt.", "px.")
exec(viz_code) # Execute the visualization code
st.plotly_chart(fig)
else:
st.warning("Could not generate a graph. Try a different query.")
except Exception as e:
st.error(f"Error during Data Visualization: {e}")
except Exception as e:
st.error(f"An error occurred during processing: {e}")
else:
st.info("Please load a dataset to start analysis.")
|