Initial commit - Pregnancy RAG Chatbot

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Pregnancy RAG Chatbot
+emoji: 📈
+colorFrom: indigo
+colorTo: pink
+sdk: gradio
+sdk_version: 5.35.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: Pregnancy Risk Assessment AI Chatbot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,477 @@
+import gradio as gr
+import os
+import sys
+from datetime import datetime
+import traceback
+
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+
+from rag_functions import get_direct_answer, get_answer_with_query_engine
+from utils import get_index
+print("✅ Successfully imported RAG functions")
+
+class PregnancyRiskAgent:
+    def __init__(self):
+        self.conversation_history = []  
+        self.current_symptoms = {}
+        self.risk_assessment_done = False
+        self.user_context = {}  
+        self.last_user_query = ""  
+        
+        
+        self.symptom_questions = [
+            "Are you currently experiencing any unusual bleeding or discharge?",
+            "How would you describe your baby's movements today compared to yesterday?",
+            "Have you had any headaches that won't go away or that affect your vision?",
+            "Do you feel any pressure or pain in your pelvis or lower back?",
+            "Are you experiencing any other symptoms? (If yes, please describe briefly)"
+        ]
+        
+        self.current_question_index = 0
+        self.waiting_for_first_response = True
+        
+    def add_to_conversation_history(self, role, message):
+        self.conversation_history.append({
+            "role": role,
+            "message": message,
+            "timestamp": datetime.now().isoformat()
+        })
+        
+        
+        if len(self.conversation_history) > 20:
+            self.conversation_history = self.conversation_history[-20:]
+    
+    def get_conversation_context(self):
+        context_parts = []
+        
+        recent_history = self.conversation_history[-10:]
+        
+        for entry in recent_history:
+            if entry["role"] == "user":
+                context_parts.append(f"User: {entry['message']}")
+            else:
+                context_parts.append(f"Assistant: {entry['message'][:200]}...")
+        
+        return "\n".join(context_parts)
+    
+    def is_follow_up_question(self, user_input):
+        follow_up_indicators = [
+            "what about", "can you explain", "what does", "why", "how", 
+            "tell me more", "what should i", "is it normal", "should i be worried",
+            "what if", "when should", "how long", "what causes", "is this"
+        ]
+        
+        user_lower = user_input.lower()
+        return any(indicator in user_lower for indicator in follow_up_indicators)
+    
+    def process_user_input(self, user_input, chat_history):
+        try:
+            self.last_user_query = user_input
+            self.add_to_conversation_history("user", user_input)
+            
+            
+            if self.waiting_for_first_response:
+                self.current_symptoms[f"question_0"] = user_input
+                self.waiting_for_first_response = False
+                self.current_question_index = 1
+                
+                if self.current_question_index < len(self.symptom_questions):
+                    bot_response = f"{self.symptom_questions[self.current_question_index]}"
+                else:
+                    bot_response = self.provide_risk_assessment()
+                    self.risk_assessment_done = True
+                
+                self.add_to_conversation_history("assistant", bot_response)
+                return bot_response
+            
+            
+            elif self.current_question_index < len(self.symptom_questions) and not self.risk_assessment_done:
+                self.current_symptoms[f"question_{self.current_question_index}"] = user_input
+                self.current_question_index += 1
+                
+                if self.current_question_index < len(self.symptom_questions):
+                    bot_response = f"{self.symptom_questions[self.current_question_index]}"
+                else:
+                    bot_response = self.provide_risk_assessment()
+                    self.risk_assessment_done = True
+                
+                self.add_to_conversation_history("assistant", bot_response)
+                return bot_response
+            
+            
+            else:
+                bot_response = self.handle_follow_up_conversation(user_input)
+                self.add_to_conversation_history("assistant", bot_response)
+                return bot_response
+                
+        except Exception as e:
+            print(f"❌ Error in process_user_input: {e}")
+            traceback.print_exc()
+            error_response = "I encountered an error. Please try again or consult your healthcare provider."
+            self.add_to_conversation_history("assistant", error_response)
+            return error_response
+    
+    def handle_follow_up_conversation(self, user_input):
+        try:
+            print(f"🔍 Processing follow-up question: {user_input}")
+            
+            symptom_summary = self.create_symptom_summary()
+            conversation_context = self.get_conversation_context()
+            
+            if any(word in user_input.lower() for word in ["last", "previous", "what did i ask", "my question"]):
+                if self.last_user_query:
+                    return f"Your last question was: \"{self.last_user_query}\"\n\nWould you like me to elaborate on that topic or do you have a different question?"
+                else:
+                    return "I don't have a record of your previous question. Could you please rephrase what you'd like to know?"
+            
+            rag_response = get_direct_answer(user_input, symptom_summary, conversation_context=conversation_context, is_risk_assessment=False)
+            
+            if "Error" in rag_response or len(rag_response) < 50:
+                print("🔄 Trying alternative method...")
+                rag_response = get_answer_with_query_engine(user_input)
+            
+            bot_response = f"""Based on your symptoms and medical literature:
+
+{rag_response}"""
+            
+            return bot_response
+            
+        except Exception as e:
+            print(f"❌ Error in follow-up conversation: {e}")
+            return "I encountered an error processing your question. Could you please rephrase it or consult your healthcare provider?"
+        
+    def create_symptom_summary(self):
+        if not self.current_symptoms:
+            return "No specific symptoms reported yet"
+            
+        summary_parts = []
+        for i, (key, response) in enumerate(self.current_symptoms.items()):
+            if i < len(self.symptom_questions):
+                question = self.symptom_questions[i]
+                summary_parts.append(f"{question}: {response}")
+        return "\n".join(summary_parts)
+
+    def parse_risk_level(self, text):
+        import re
+        
+        patterns = [
+            r'\*\*Risk Level:\*\*\s*(Low|Medium|High)',  
+            r'Risk Level:\s*\*\*(Low|Medium|High)\*\*',  
+            r'Risk Level:\s*(Low|Medium|High)',          
+            r'\*\*Risk Level:\*\*\s*<(Low|Medium|High)>', 
+            r'Risk Level.*?<(Low|Medium|High)>',          
+        ]
+        
+        for pattern in patterns:
+            match = re.search(pattern, text, re.IGNORECASE)
+            if match:
+                risk_level = match.group(1).capitalize()
+                print(f"✅ Successfully parsed risk level: {risk_level}")
+                return risk_level
+        
+        print(f"❌ Could not parse risk level from: {text[:200]}...")
+        return None
+
+    def provide_risk_assessment(self):
+        all_symptoms = self.create_symptom_summary()
+        
+        rag_query = f"Analyze these pregnancy symptoms for risk assessment:\n{all_symptoms}\n\nProvide risk level and medical recommendations."
+        detailed_analysis = get_direct_answer(rag_query, all_symptoms, is_risk_assessment=True)
+
+        print(f"🔍 RAG Response: {detailed_analysis[:300]}...")
+        
+        llm_risk_level = self.parse_risk_level(detailed_analysis)
+        
+        if llm_risk_level:
+            risk_level = llm_risk_level
+            
+            if risk_level == "Low":
+                action = "✅ Continue routine prenatal care and self-monitoring"
+            elif risk_level == "Medium":
+                action = "⚠️ Contact your doctor within 24 hours"
+            elif risk_level == "High":
+                action = "🚨 Immediate visit to ER or OB emergency care required"
+        else:
+            print("⚠️ RAG assessment failed, using fallback")
+            risk_level = "Medium"
+            action = "⚠️ Contact your doctor within 24 hours"
+
+        symptom_list = []
+        for i, (key, symptom) in enumerate(self.current_symptoms.items()):
+            question = self.symptom_questions[i] if i < len(self.symptom_questions) else f"Question {i+1}"
+            symptom_list.append(f"• **{question}**: {symptom}")
+        
+        assessment = f"""
+## 🏥 **Risk Assessment Complete**
+
+**Risk Level: {risk_level}**
+**Recommended Action: {action}**
+
+### 📋 **Your Reported Symptoms:**
+{chr(10).join(symptom_list)}
+
+### 🔬 **Medical Analysis:**
+{detailed_analysis}
+
+### 💡 **Next Steps:**
+- Follow the recommended action above
+- Keep monitoring your symptoms
+- Contact your healthcare provider if symptoms worsen
+- Feel free to ask me any follow-up questions about pregnancy health
+
+"""
+        return assessment
+    
+    def reset_conversation(self):
+        self.conversation_history = []
+        self.current_symptoms = {}
+        self.current_question_index = 0
+        self.risk_assessment_done = False
+        self.waiting_for_first_response = True
+        self.user_context = {}
+        self.last_user_query = ""
+        return get_welcome_message()
+
+def get_welcome_message():
+    return """Hello! I'm here to help assess pregnancy-related symptoms and provide risk insights based on medical literature.
+
+I'll ask you a few important questions about your current symptoms, then provide a risk assessment and recommendations. After that, feel free to ask any follow-up questions!
+
+**To get started, please tell me:**
+Are you currently experiencing any unusual bleeding or discharge?
+
+---
+⚠️ **Important**: This tool is for informational purposes only and should not replace professional medical care. In case of emergency, contact your healthcare provider immediately."""
+
+
+def create_new_agent():
+    
+    return PregnancyRiskAgent()
+
+
+agent = create_new_agent()
+
+def chat_interface_with_reset(user_input, history):
+    global agent
+    
+    if user_input.lower() in ["reset", "restart", "new assessment"]:
+        agent = create_new_agent()
+        return get_welcome_message()
+    
+    response = agent.process_user_input(user_input, history)
+    return response
+
+def reset_chat():
+    global agent
+    agent = create_new_agent()
+    return [{"role": "assistant", "content": get_welcome_message()}], ""
+
+
+
+custom_css = """
+body, .gradio-container {
+    color: yellow !important;
+}
+
+.header {
+    background: linear-gradient(135deg, #ff9a9e 0%, #fecfef 100%);
+    padding: 2rem;
+    border-radius: 1rem;
+    text-align: center;
+    margin-bottom: 2rem;
+    box-shadow: 0 4px 12px rgba(0,0,0,0.1);
+}
+
+.header h1 {
+    color: black !important;
+    margin-bottom: 0.5rem;
+    font-size: 2.5rem;
+}
+
+.header p {
+    color: black !important;
+    font-size: 1.1rem;
+    margin: 0.5rem 0;
+}
+
+.warning {
+    background-color: #fff4e6;
+    border-left: 6px solid #ff7f00;
+    padding: 15px;
+    border-radius: 5px;
+    margin: 10px 0;
+}
+
+.warning h3 {
+    color: black !important;
+    margin-top: 0;
+}
+
+.warning p {
+    color: black !important;
+    line-height: 1.6;
+}
+
+div[style*="background-color: #e8f5e8"] {
+    color: black !important;
+}
+
+div[style*="background-color: #e8f5e8"] h3 {
+    color: black !important;
+}
+
+div[style*="background-color: #e8f5e8"] li {
+    color: black !important;
+}
+
+.chatbot {
+    color: black !important;
+}
+
+.message {
+    color: black !important;
+}
+
+/* Hide Gradio footer elements */
+.footer {
+    display: none !important;
+}
+
+.gradio-container .footer {
+    display: none !important;
+}
+
+footer {
+    display: none !important;
+}
+
+.api-docs {
+    display: none !important;
+}
+
+.built-with {
+    display: none !important;
+}
+
+.gradio-container > .built-with {
+    display: none !important;
+}
+
+.settings {
+    display: none !important;
+}
+
+div[class*="footer"] {
+    display: none !important;
+}
+
+div[class*="built"] {
+    display: none !important;
+}
+
+*:contains("Built with Gradio") {
+    display: none !important;
+}
+
+*:contains("Use via API") {
+    display: none !important;
+}
+
+*:contains("Settings") {
+    display: none !important;
+}
+"""
+
+
+with gr.Blocks(css=custom_css) as demo:
+    gr.HTML("""
+    <div class="header">
+        <h1>🤱 Pregnancy RAG Chatbot</h1>
+        <p><strong style="color: black !important;">Proactive RAG-powered pregnancy risk management</strong></p>
+    </div>
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.HTML("""
+            <div class="warning">
+                <h3>⚠️ Medical Disclaimer</h3>
+                <p>This AI assistant provides information based on medical literature but is NOT a substitute for professional medical advice, diagnosis, or treatment.</p>
+                <p><strong style="color: black !important;">In emergencies, call emergency services immediately.</strong></p>
+            </div>
+            """)
+    
+    
+    chatbot = gr.ChatInterface(
+        fn=chat_interface_with_reset,
+        chatbot=gr.Chatbot(
+            value=[{"role": "assistant", "content": get_welcome_message()}],
+            show_label=False,
+            type='messages'
+        ),
+        textbox=gr.Textbox(
+            placeholder="Type your response here...", 
+            show_label=False,
+            max_length=1000,
+            submit_btn=True
+        )
+    )
+
+    with gr.Row():
+        reset_btn = gr.Button("🔄 Start New Assessment", variant="secondary")
+
+        reset_btn.click(
+            fn=reset_chat,
+            outputs=[chatbot.chatbot, chatbot.textbox],
+            show_progress=False
+        )
+
+
+def check_groq_connection():
+    try:
+        from backend.utils import llm
+        test_response = llm.complete("Hello")
+        print("✅ Groq connection successful")
+        return True
+    except Exception as e:
+        print(f"❌ Groq connection failed: {e}")
+        return False
+
+
+def refresh_page():
+    """Force a complete page refresh"""
+    return None
+
+
+
+if __name__ == "__main__":
+    print("🚀 Starting GraviLog Pregnancy Risk Assessment Agent...")
+    check_groq_connection()
+    
+    
+    is_hf_space = os.getenv('SPACE_ID') is not None
+    
+    if is_hf_space:
+        print("📍 Running on Hugging Face Spaces")
+        print("📍 Each page refresh will start a new conversation")
+        demo.queue().launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            share=False,  
+            debug=False   
+        )
+    else:
+        print("📍 Running locally")
+        print("📍 Using Groq API for LLM processing")
+        print("📍 Make sure your GROQ_API_KEY is set in environment variables")
+        print("📍 Make sure your Pinecone index is set up and populated")
+        
+        demo.queue().launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            share=True,
+            debug=True,
+            show_error=True
+        )

rag_functions.py

@@ -0,0 +1,246 @@
+import os
+import Stemmer
+import requests
+from utils import get_and_chunk_documents, llm, embed_model, get_index
+from utils import Settings 
+from llama_index.retrievers.bm25 import BM25Retriever
+from llama_index.core.postprocessor import SentenceTransformerRerank
+from llama_index.core.query_engine import RetrieverQueryEngine
+from llama_index.core.response_synthesizers import get_response_synthesizer
+from llama_index.core.settings import Settings
+from llama_index.core import VectorStoreIndex
+from llama_index.core.llms import ChatMessage
+from llama_index.core.retrievers import QueryFusionRetriever
+import json
+
+
+Settings.llm = llm
+Settings.embed_model = embed_model
+
+
+index = get_index()
+hybrid_retriever = None
+vector_retriever = None
+bm25_retriever = None
+
+if index:
+    try:
+        
+        vector_retriever = index.as_retriever(similarity_top_k=15)
+        print("✅ Vector retriever initialized successfully")
+        
+        
+        all_nodes = index.docstore.docs
+        if len(all_nodes) == 0:
+            print("⚠️ Warning: No documents found in index, skipping BM25 retriever")
+            hybrid_retriever = vector_retriever  
+        else:
+            
+            has_text_content = False
+            for node_id, node in all_nodes.items():
+                if hasattr(node, 'text') and node.text and node.text.strip():
+                    has_text_content = True
+                    break
+            
+            if not has_text_content:
+                print("⚠️ Warning: No text content found in documents, skipping BM25 retriever")
+                hybrid_retriever = vector_retriever  
+            else:
+                try:
+                    
+                    print("🔄 Creating BM25 retriever...")
+                    bm25_retriever = BM25Retriever.from_defaults(
+                        docstore=index.docstore, 
+                        similarity_top_k=15,
+                        verbose=False
+                    )
+                    print("✅ BM25 retriever initialized successfully")
+                    
+                    
+                    hybrid_retriever = QueryFusionRetriever(
+                        retrievers=[vector_retriever, bm25_retriever],
+                        similarity_top_k=20,
+                        num_queries=1,  
+                        mode="reciprocal_rerank",  
+                        use_async=False,
+                    )
+                    print("✅ Hybrid retriever initialized successfully")
+                    
+                except Exception as e:
+                    print(f"❌ Warning: Could not initialize BM25 retriever: {e}")
+                    print("🔄 Falling back to vector-only retrieval")
+                    hybrid_retriever = vector_retriever
+        
+    except Exception as e:
+        print(f"❌ Warning: Could not initialize retrievers: {e}")
+        hybrid_retriever = None
+        vector_retriever = None
+        bm25_retriever = None
+else:
+    print("❌ Warning: Could not initialize retrievers - index is None")
+
+def call_groq_api(prompt):
+    """Call Groq API instead of LM Studio"""
+    try:
+        
+        response = Settings.llm.complete(prompt)
+        return str(response)
+    except Exception as e:
+        print(f"❌ Groq API call failed: {e}")
+        raise e
+
+def get_direct_answer(question, symptom_summary, conversation_context="", max_context_nodes=8, is_risk_assessment=True):
+    """Get answer using hybrid retriever with retrieved context"""
+    
+    print(f"🎯 Processing question: {question}")
+    
+    if not hybrid_retriever:
+        return "Error: Retriever not available. Please check if documents are properly loaded in the index."
+    
+    try:
+        
+        print("🔍 Retrieving with available retrieval method...")
+        retrieved_nodes = hybrid_retriever.retrieve(question)
+        print(f"📊 Retrieved {len(retrieved_nodes)} nodes")
+        
+    except Exception as e:
+        print(f"❌ Retrieval failed: {e}")
+        return f"Error during document retrieval: {e}. Please check your document index."
+    
+    if not retrieved_nodes:
+        return "No relevant documents found for this question. Please ensure your medical knowledge base is properly loaded and consult your healthcare provider for medical advice."
+    
+    
+    try:
+        reranker = SentenceTransformerRerank(
+            model='cross-encoder/ms-marco-MiniLM-L-2-v2',
+            top_n=max_context_nodes,
+        )
+        
+        reranked_nodes = reranker.postprocess_nodes(retrieved_nodes, query_str=question)
+        print(f"🎯 After reranking: {len(reranked_nodes)} nodes")
+        
+    except Exception as e:
+        print(f"❌ Reranking failed: {e}, using original nodes")
+        reranked_nodes = retrieved_nodes[:max_context_nodes]
+    
+    
+    filtered_nodes = []
+    pregnancy_keywords = ['pregnancy', 'preeclampsia', 'gestational', 'trimester', 'fetal', 'bleeding', 'contractions', 'prenatal']
+    
+    for node in reranked_nodes:
+        node_text = node.get_text().lower()
+        if any(keyword in node_text for keyword in pregnancy_keywords):
+            filtered_nodes.append(node)
+    
+    if filtered_nodes:
+        reranked_nodes = filtered_nodes[:max_context_nodes]
+        print(f"🔍 After pregnancy keyword filtering: {len(reranked_nodes)} nodes")
+    else:
+        print("⚠️ No pregnancy-related content found, using original nodes")
+    
+    
+    context_chunks = []
+    total_chars = 0
+    max_context_chars = 6000  
+    
+    for node in reranked_nodes:
+        node_text = node.get_text()
+        if total_chars + len(node_text) <= max_context_chars:
+            context_chunks.append(node_text)
+            total_chars += len(node_text)
+        else:
+            remaining_chars = max_context_chars - total_chars
+            if remaining_chars > 100:
+                context_chunks.append(node_text[:remaining_chars] + "...")
+            break
+    
+    context_text = "\n\n---\n\n".join(context_chunks)
+    
+    
+    if is_risk_assessment:
+        prompt = f"""You are the GraviLog Pregnancy Risk Assessment Agent. Use ONLY the context below—do not invent or add any new medical facts.
+
+    SYMPTOM RESPONSES:
+    {symptom_summary}
+
+    MEDICAL KNOWLEDGE:
+    {context_text}
+
+    Respond ONLY in this exact format (no extra text):
+
+    🏥 Risk Assessment Complete  
+    **Risk Level:** <Low/Medium/High>  
+    **Recommended Action:** <from KB's Risk Output Labels>  
+
+    🔬 Rationale:  
+    <One or two sentences citing which bullet(s) from the KB triggered your risk level.>"""
+
+    else:
+        
+        prompt = f"""You are a pregnancy health assistant. Based on the medical knowledge below, answer the user's question about pregnancy symptoms and conditions.
+
+    USER QUESTION: {question}
+
+    CONVERSATION CONTEXT:
+    {conversation_context}
+
+    CURRENT SYMPTOMS REPORTED:
+    {symptom_summary}
+
+    MEDICAL KNOWLEDGE:
+    {context_text}
+
+    Provide a clear, informative answer based on the medical knowledge. Always mention if symptoms require medical attention and provide risk level (Low/Medium/High) when relevant."""
+    
+    try:
+        print("🤖 Generating response with Groq API...")
+        response_text = call_groq_api(prompt)
+        return response_text
+        
+    except Exception as e:
+        print(f"❌ LLM response failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return f"Error generating response: {e}"
+
+def get_answer_with_query_engine(question):
+    """Alternative approach using LlamaIndex query engine with hybrid retrieval"""
+    try:
+        print(f"🎯 Processing question with query engine: {question}")
+        
+        if index is None:
+            return "Error: Could not load index"
+        
+        
+        if hybrid_retriever:
+            query_engine = RetrieverQueryEngine.from_args(
+                retriever=hybrid_retriever,
+                response_synthesizer=get_response_synthesizer(
+                    response_mode="compact",
+                    use_async=False
+                ),
+                node_postprocessors=[
+                    SentenceTransformerRerank(
+                        model='cross-encoder/ms-marco-MiniLM-L-2-v2',
+                        top_n=5
+                    )
+                ]
+            )
+        else:
+            
+            query_engine = index.as_query_engine(
+                similarity_top_k=10,
+                response_mode="compact"
+            )
+        
+        print("🤖 Querying with engine...")
+        response = query_engine.query(question)
+        
+        return str(response)
+        
+    except Exception as e:
+        print(f"❌ Query engine failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return f"Error with query engine: {e}. Please check your setup and try again."

utils.py

@@ -0,0 +1,184 @@
+import os
+from dotenv import load_dotenv
+from pinecone import Pinecone, ServerlessSpec
+from llama_index.core import (SimpleDirectoryReader,Document, VectorStoreIndex, StorageContext, load_index_from_storage)
+from llama_index.core.node_parser import SemanticSplitterNodeParser
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.readers.file import CSVReader
+from llama_index.vector_stores.pinecone import PineconeVectorStore
+from llama_index.core.settings import Settings
+from llama_index.llms.groq import Groq
+
+
+
+load_dotenv()
+
+
+embed_model = HuggingFaceEmbedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
+llm = Groq(
+    model="llama-3.1-8b-instant",  
+    api_key=os.getenv("GROQ_API_KEY"),
+    max_tokens=500,
+    temperature=0.1
+)
+
+
+Settings.embed_model = embed_model
+Settings.llm = llm
+
+
+pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY"))
+index_name = os.getenv("PINECONE_INDEX")
+
+def get_vector_store():
+    
+    pinecone_index = pc.Index(index_name)
+    return PineconeVectorStore(pinecone_index=pinecone_index)
+
+def get_storage_context(for_rebuild=False):
+    
+    vector_store = get_vector_store()
+    persist_dir = "./storage"
+    
+    if for_rebuild or not os.path.exists(persist_dir):
+    
+        return StorageContext.from_defaults(vector_store=vector_store)
+    else:
+    
+        return StorageContext.from_defaults(
+            vector_store=vector_store,
+            persist_dir=persist_dir
+        )
+
+
+
+
+
+def get_and_chunk_documents():
+
+    try:
+
+        file_extractor = {".csv": CSVReader()}
+
+
+        documents = SimpleDirectoryReader(
+            "../knowledge_base", 
+            file_extractor=file_extractor
+        ).load_data()
+
+        print(f"📖 Loaded {len(documents)} documents")
+
+        node_parser = SemanticSplitterNodeParser(
+            buffer_size=1, 
+            breakpoint_percentile_threshold=95, 
+            embed_model=embed_model
+        )
+
+        nodes = node_parser.get_nodes_from_documents(documents)
+        print(f"📄 Created {len(nodes)} document chunks")
+        return nodes
+
+    except Exception as e:
+        print(f"❌ Error loading documents: {e}")
+        return []
+
+
+def get_index():
+
+    try:
+        storage_context = get_storage_context()
+
+        return load_index_from_storage(storage_context)
+    except Exception as e:
+        print(f"⚠️ Local storage not found, creating index from existing Pinecone data...")
+        try:
+
+            vector_store = get_vector_store()
+            storage_context = get_storage_context()
+            index = VectorStoreIndex.from_vector_store(
+                vector_store=vector_store,
+                storage_context=storage_context
+            )
+            return index
+        except Exception as e2:
+            print(f"❌ Error creating index from vector store: {e2}")
+            return None
+
+def check_index_status():
+
+    try:
+        pinecone_index = pc.Index(index_name)
+        stats = pinecone_index.describe_index_stats()
+        vector_count = stats.get('total_vector_count', 0)
+        
+        if vector_count > 0:
+            print(f"✅ Index found with {vector_count} vectors")
+            return True
+        else:
+            print("❌ Index exists but is empty")
+            return False
+    except Exception as e:
+        print(f"❌ Error checking index: {e}")
+        return False
+    
+
+
+def clear_pinecone_index():
+    """Delete all vectors from Pinecone index"""
+    try:
+        pinecone_index = pc.Index(index_name)
+        
+
+        stats = pinecone_index.describe_index_stats()
+        vector_count = stats.get('total_vector_count', 0)
+        print(f"🗑️ Current vectors in index: {vector_count}")
+        
+        if vector_count > 0:
+
+            pinecone_index.delete(delete_all=True)
+            print("✅ All vectors deleted from Pinecone index")
+        else:
+            print("ℹ️ Index is already empty")
+            
+        return True
+        
+    except Exception as e:
+        print(f"❌ Error clearing index: {e}")
+        return False
+
+def rebuild_index():
+    """Clear old data and rebuild index with new CSV processing"""
+    try:
+        print("🔄 Starting index rebuild process...")
+        
+
+        if not clear_pinecone_index():
+            print("❌ Failed to clear index, aborting rebuild")
+            return None
+        
+
+        import shutil
+        if os.path.exists("./storage"):
+            shutil.rmtree("./storage")
+            print("🗑️ Cleared local storage")
+        
+
+        nodes = get_and_chunk_documents()
+        
+        if not nodes:
+            print("❌ No nodes created, cannot rebuild index")
+            return None
+        
+
+        storage_context = get_storage_context(for_rebuild=True)
+        index = VectorStoreIndex(nodes, storage_context=storage_context)
+        
+
+        index.storage_context.persist(persist_dir="./storage")
+        
+        print(f"✅ Index rebuilt successfully with {len(nodes)} nodes")
+        return index
+        
+    except Exception as e:
+        print(f"❌ Error rebuilding index: {e}")
+        return None