added exception handling

pages/Load_Documents.py

@@ -1,14 +1,40 @@
 import os
+import openai
 from utils._admin_util import create_embeddings, create_vector_store, read_pdf_data, split_data
 import streamlit as st
 from dotenv import load_dotenv
+import requests
+
+def validate_api_key(api_key):
+    """Test if the API key is valid"""
+    try:
+       # Make a small test request to OpenAI
+        client = openai.OpenAI(api_key=api_key)
+        client.embeddings.create(input="test", model="text-embedding-ada-002")
+        return True
+    except openai.AuthenticationError:
+        st.error("❌ Invalid API key")
+        return False
+    except openai.PermissionDeniedError:
+        st.error("❌ Permission denied. Please check your API key's permissions")
+        return False
+    except Exception as e:
+        st.error(f"❌ API key validation error: {str(e)}")
+        return False
 
 def main():
     load_dotenv()
     
-    # Add API key verification at the start
-    if not os.getenv("OPENAI_API_KEY"):
-        st.error("OpenAI API key not found! Please ensure it's set in the environment variables.")
+    # Add detailed API key verification
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        st.error("❌ OpenAI API key not found! Please ensure it's set in the environment variables.")
+        st.info("To set up your API key:")
+        st.code("1. Go to Hugging Face Space settings\n2. Add OPENAI_API_KEY in Repository Secrets")
+        st.stop()
+    
+    # Validate the API key
+    if not validate_api_key(api_key):
         st.stop()
 
     st.set_page_config(page_title="Dump PDFs to QDrant - Vector Store")
@@ -24,31 +50,55 @@ def main():
                 
                 # Process each PDF file
                 for pdf in uploaded_files:
-                    st.write(f"Processing: {pdf.name}")
-                    
-                    # Extract text from PDF
-                    text = read_pdf_data(pdf)
-                    st.write(f"👉 Reading {pdf.name} done")
-
-                    # Create chunks for this PDF
-                    chunks = split_data(text)
-                    all_chunks.extend(chunks)
-                    st.write(f"👉 Splitting {pdf.name} into chunks done")
-
-                # Create embeddings once for all chunks
-                st.write("Creating embeddings...")
-                embeddings = create_embeddings()
-                st.write("👉 Creating embeddings instance done")
+                    try:
+                        st.write(f"Processing: {pdf.name}")
+                        
+                        # Extract text from PDF
+                        text = read_pdf_data(pdf)
+                        st.write(f"👉 Reading {pdf.name} done")
+
+                        # Create chunks for this PDF
+                        chunks = split_data(text)
+                        all_chunks.extend(chunks)
+                        st.write(f"👉 Splitting {pdf.name} into chunks done")
+                    except Exception as e:
+                        st.error(f"❌ Error processing {pdf.name}: {str(e)}")
+                        continue
+
+                if not all_chunks:
+                    st.error("❌ No valid chunks were created from the PDFs")
+                    st.stop()
+
+                # Create embeddings with progress tracking
+                try:
+                    st.write("Creating embeddings...")
+                    embeddings = create_embeddings()
+                    st.write("👉 Creating embeddings instance done")
+                except openai.RateLimitError:
+                    st.error("❌ Rate limit exceeded. Please try again later.")
+                    st.stop()
+                except Exception as e:
+                    st.error(f"❌ Error creating embeddings: {str(e)}")
+                    st.stop()
 
                 # Create vector store with all chunks
-                vector_store = create_vector_store(embeddings, all_chunks)
-                st.session_state.vector_store = vector_store
+                try:
+                    vector_store = create_vector_store(embeddings, all_chunks)
+                    st.session_state.vector_store = vector_store
+                except Exception as e:
+                    st.error(f"❌ Error creating vector store: {str(e)}")
+                    st.stop()
                 
-                st.success(f"Successfully processed {len(uploaded_files)} files and pushed embeddings to Qdrant")
+                st.success(f"✅ Successfully processed {len(uploaded_files)} files and pushed embeddings to Qdrant")
                 st.write(f"Total chunks created: {len(all_chunks)}")
+
+        except requests.exceptions.RequestException as e:
+            st.error(f"❌ Network error: {str(e)}")
+            st.error("Please check your internet connection and try again.")
         except Exception as e:
-            st.error(f"An error occurred: {str(e)}")
+            st.error(f"❌ An unexpected error occurred: {str(e)}")
             st.error("Please check your API key and permissions.")
+            st.error("If the problem persists, please contact support.")
 
 if __name__ == '__main__':
     main()

utils/_admin_util.py

@@ -1,3 +1,4 @@
+import os
 import tiktoken
 import PyPDF2
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -9,6 +10,8 @@ from langchain_core.runnables import RunnablePassthrough
 from langchain_core.output_parsers import StrOutputParser
 from langchain_openai import ChatOpenAI
 from langchain_core.prompts import ChatPromptTemplate
+import streamlit as st
+
 
 
 HUMAN_TEMPLATE = """
@@ -21,120 +24,166 @@ QUERY:
 Use the provide context to answer the provided user query. Only use the provided context to answer the query. If you do not know the answer, or it's not contained in the provided context response with "I don't know"
 """
 
+# Define the system prompt for categorization
+CATEGORY_PROMPT = """You are a ticket categorization system. Categorize the following query into exactly one of these categories:
+    - HR Support: For queries about employment, benefits, leaves, workplace policies, etc.
+    - IT Support: For queries about software, hardware, network, system access, etc.
+    - Transportation Support: For queries about company transport, parking, vehicle maintenance, etc.
+    - Other: For queries that do not fit into the above categories.
+    Respond with ONLY the category name, nothing else.
+    
+    Query: {query}
+    """
+
+def check_api_key():
+    """Verify that the API key is set and valid"""
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError("OpenAI API key not found in environment variables")
+    return api_key
+
 #Read PDF data
 def read_pdf_data(pdf_file):
-    # Create PDF reader object
-    pdf_reader = PyPDF2.PdfReader(pdf_file)
-    
-    # Extract text from each page
-    text = ""
-    for page in pdf_reader.pages:
-        text += page.extract_text()
-    return text
+    try:
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        text = ""
+        for page in pdf_reader.pages:
+            text += page.extract_text()
+        if not text.strip():
+            raise ValueError("No text extracted from PDF")
+        return text
+    except Exception as e:
+        raise Exception(f"Error reading PDF: {str(e)}")
 
 def tiktoken_len(text):
-    tokens = tiktoken.encoding_for_model("gpt-4").encode(
-        text,
-    )
-    return len(tokens)
+    try:
+        tokens = tiktoken.encoding_for_model("gpt-4").encode(text)
+        return len(tokens)
+    except Exception as e:
+        raise Exception(f"Error in token calculation: {str(e)}")
 
 #Split data into chunks
 def split_data(text):
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size = 100,
-        chunk_overlap = 0,
-        length_function = tiktoken_len,
-    )   
-    chunks = text_splitter.split_text(text)
-    return chunks
+    try:
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=500,  # Increased for better context
+            chunk_overlap=50,  # Added overlap for better continuity
+            length_function=tiktoken_len,
+            separators=["\n\n", "\n", " ", ""]
+        )   
+        chunks = text_splitter.split_text(text)
+        if not chunks:
+            raise ValueError("Text splitting produced no chunks")
+        return chunks
+    except Exception as e:
+        raise Exception(f"Error splitting text: {str(e)}")
 
 #Create embeddings instance
 
 def create_embeddings():
-    embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
-    return embedding_model
+    try:
+        api_key = check_api_key()
+        embedding_model = OpenAIEmbeddings(
+            model="text-embedding-3-small",
+            openai_api_key=api_key,
+            show_progress_bar=True
+        )
+        return embedding_model
+    except Exception as e:
+        raise Exception(f"Error creating embeddings model: {str(e)}")
 
 
 # Create a vector database using Qdrant
 def create_vector_store(embedding_model, chunks):
-    embedding_dim = 1536  # YOUR ANSWER HERE
-    client = QdrantClient(":memory:")
-    client.create_collection(
-        collection_name="lcel_doc_v2",
-        vectors_config=VectorParams(size=embedding_dim, distance=Distance.COSINE),
-    )   
-    vector_store = QdrantVectorStore(
-        client=client,
-        collection_name="lcel_doc_v2",
-        embedding=embedding_model,
-    )
-    _ = vector_store.add_texts(texts=chunks)
-    return vector_store
+    try:
+        embedding_dim = 1536
+        client = QdrantClient(":memory:")  # Consider using persistent storage for production
+        
+        # Create collection with error handling
+        try:
+            client.create_collection(
+                collection_name="lcel_doc_v2",
+                vectors_config=VectorParams(size=embedding_dim, distance=Distance.COSINE),
+            )
+        except Exception as e:
+            raise Exception(f"Error creating Qdrant collection: {str(e)}")
+
+        vector_store = QdrantVectorStore(
+            client=client,
+            collection_name="lcel_doc_v2",
+            embedding=embedding_model,
+        )
+        
+        # Add texts with progress tracking
+        try:
+            _ = vector_store.add_texts(texts=chunks)
+        except Exception as e:
+            raise Exception(f"Error adding texts to vector store: {str(e)}")
+            
+        return vector_store
+    except Exception as e:
+        raise Exception(f"Error in vector store creation: {str(e)}")
 
 # create RAG
 def create_rag(vector_store):
-    # Initialize OpenAI chat model with a valid model name
-    openai_chat_model = ChatOpenAI(model="gpt-3.5-turbo")
-    
-    # Create chat prompt template
-    chat_prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are a helpful assistant that answers questions based on the provided context."),
-        ("human", HUMAN_TEMPLATE)
-    ])
-    
-    # Set up retriever with configurable k
-    retriever = vector_store.as_retriever(search_kwargs={"k": 3})
-    
-    # Create RAG pipeline
-    simple_rag = (
-        {"context": retriever, "query": RunnablePassthrough()}
-        | chat_prompt
-        | openai_chat_model
-        | StrOutputParser() 
-    ) 
-    
-    return simple_rag
+    try:
+        api_key = check_api_key()
+        openai_chat_model = ChatOpenAI(
+            model="gpt-3.5-turbo",
+            openai_api_key=api_key,
+            temperature=0.7
+        )
+        
+        chat_prompt = ChatPromptTemplate.from_messages([
+            ("system", "You are a helpful assistant that answers questions based on the provided context."),
+            ("human", HUMAN_TEMPLATE)
+        ])
+        
+        retriever = vector_store.as_retriever(search_kwargs={"k": 3})
+        
+        simple_rag = (
+            {"context": retriever, "query": RunnablePassthrough()}
+            | chat_prompt
+            | openai_chat_model
+            | StrOutputParser() 
+        ) 
+        
+        return simple_rag
+    except Exception as e:
+        raise Exception(f"Error creating RAG chain: {str(e)}")
 
 # Invoke RAG
 def invoke_rag(vector_store, query):
-    rag_chain = create_rag(vector_store)
-    response = rag_chain.invoke(query)
-    return response
+    try:
+        rag_chain = create_rag(vector_store)
+        response = rag_chain.invoke(query)
+        return response
+    except Exception as e:
+        raise Exception(f"Error invoking RAG chain: {str(e)}")
 
 
 def get_ticket_category(query):
-    # Define the system prompt for categorization
-    CATEGORY_PROMPT = """You are a ticket categorization system. Categorize the following query into exactly one of these categories:
-    - HR Support: For queries about employment, benefits, leaves, workplace policies, etc.
-    - IT Support: For queries about software, hardware, network, system access, etc.
-    - Transportation Support: For queries about company transport, parking, vehicle maintenance, etc.
-    - Other: For queries that do not fit into the above categories.
-    Respond with ONLY the category name, nothing else.
-    
-    Query: {query}
-    """
-    
-    # Create OpenAI client for categorization
-    client = ChatOpenAI(model="gpt-3.5-turbo")
-    
-    # Create the prompt template
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", CATEGORY_PROMPT)
-    ])
-    
-    # Create the chain
-    chain = prompt | client | StrOutputParser()
-    
-    # Get the category
-    category = chain.invoke({"query": query})
-    
-    # Clean and validate the response
-    category = category.strip()
-    valid_categories = ["HR Support", "IT Support", "Transportation Support"]
-    
-    if category not in valid_categories:
-        return "Other"  # Default category if classification fails
+    try:
+        api_key = check_api_key()
+        client = ChatOpenAI(
+            model="gpt-3.5-turbo",
+            openai_api_key=api_key,
+            temperature=0
+        )
+        
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", CATEGORY_PROMPT)
+        ])
+        
+        chain = prompt | client | StrOutputParser()
+        category = chain.invoke({"query": query})
+        
+        category = category.strip()
+        valid_categories = ["HR Support", "IT Support", "Transportation Support", "Other"]
         
-    return category
+        return category if category in valid_categories else "Other"
+    except Exception as e:
+        st.error(f"Error in category classification: {str(e)}")
+        return "Other"  # Fallback category