update penyimpanan vectore_store ke supabase

app.py

@@ -1,4 +1,3 @@
-
 import streamlit as st
 import os
 import tempfile
@@ -20,7 +19,7 @@ load_dotenv()
 
 # Supabase configuration
 BUCKET_NAME = "pnp-bot-storage-archive"
-VECTOR_STORE_FILE = "vector_store.zip"
+VECTOR_STORE_PREFIX = "vector_store"  # Changed from file name to prefix
 DATA_DIR = "data"
 
 @traceable(name="Create RAG Conversational Chain")
@@ -38,7 +37,7 @@ def create_conversational_chain(vector_store):
     
     chain = ConversationalRetrievalChain.from_llm(
         llm,
-        retriever=vector_store.as_retriever(search_kwargs={"k": 10}),
+        retriever=vector_store.as_retriever(search_kwargs={"k": 6}),
         combine_docs_chain_kwargs={"prompt": sahabat_prompt},
         return_source_documents=True,
         memory=memory
@@ -60,13 +59,22 @@ def get_latest_data_timestamp(folder):
     return latest_time
 
 def get_supabase_vector_store_timestamp():
-    """Get the timestamp of vector store in Supabase storage"""
+    """Get the timestamp of vector store files in Supabase storage"""
     try:
         response = supabase.storage.from_(BUCKET_NAME).list()
+        timestamps = []
+        
         for file in response:
-            if file['name'] == VECTOR_STORE_FILE:
-                return file['updated_at']
+            if file['name'].startswith(VECTOR_STORE_PREFIX) and (
+                file['name'].endswith('.faiss') or file['name'].endswith('.pkl')
+            ):
+                timestamps.append(file['updated_at'])
+        
+        # Return the latest timestamp if both files exist
+        if len(timestamps) >= 2:
+            return max(timestamps)
         return None
+        
     except Exception as e:
         print(f"Error getting Supabase timestamp: {e}")
         return None
@@ -101,7 +109,7 @@ def main():
                     
                     # Save to Supabase instead of local storage
                     with st.spinner("Uploading vector store to Supabase..."):
-                        success = save_vector_store_to_supabase(vector_store, supabase, BUCKET_NAME, VECTOR_STORE_FILE)
+                        success = save_vector_store_to_supabase(vector_store, supabase, BUCKET_NAME, VECTOR_STORE_PREFIX)
                         if success:
                             st.success("Vector store uploaded to Supabase successfully!")
                         else:
@@ -112,7 +120,7 @@ def main():
         else:
             # Load vector store from Supabase
             with st.spinner("Loading vector store from Supabase..."):
-                vector_store = load_vector_store_from_supabase(supabase, BUCKET_NAME, VECTOR_STORE_FILE)
+                vector_store = load_vector_store_from_supabase(supabase, BUCKET_NAME, VECTOR_STORE_PREFIX)
                 if vector_store:
                     st.success("Vector store loaded from Supabase successfully!")
                 else:
@@ -122,7 +130,7 @@ def main():
         vector_store = st.session_state.get('vector_store')
         if vector_store is None:
             # Fallback: load from Supabase if not in session
-            vector_store = load_vector_store_from_supabase(supabase, BUCKET_NAME, VECTOR_STORE_FILE)
+            vector_store = load_vector_store_from_supabase(supabase, BUCKET_NAME, VECTOR_STORE_PREFIX)
     
     st.session_state['vector_store'] = vector_store
     

app/document_processor.py

@@ -6,28 +6,37 @@ import tempfile
 import zipfile
 import streamlit as st
 
-def save_vector_store_to_supabase(vector_store, supabase, bucket_name, file_name):
-    """Save vector store to Supabase storage as a zip file."""
+def save_vector_store_to_supabase(vector_store, supabase, bucket_name, file_prefix="vector_store"):
+    """Save vector store to Supabase storage as separate files."""
     try:
         with tempfile.TemporaryDirectory() as temp_dir:
             # Save vector store locally first
             local_path = os.path.join(temp_dir, "vector_store")
             vector_store.save_local(local_path)
             
-            # Create zip file
-            zip_path = os.path.join(temp_dir, "vector_store.zip")
-            with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-                for root, dirs, files in os.walk(local_path):
-                    for file in files:
-                        file_path = os.path.join(root, file)
-                        arc_name = os.path.relpath(file_path, local_path)
-                        zipf.write(file_path, arc_name)
+            # Upload index.faiss
+            faiss_file = os.path.join(local_path, "index.faiss")
+            if os.path.exists(faiss_file):
+                with open(faiss_file, 'rb') as f:
+                    supabase.storage.from_(bucket_name).upload(
+                        f"{file_prefix}_index.faiss", 
+                        f, 
+                        {"upsert": "true"}
+                    )
+                print(f"Uploaded: {file_prefix}_index.faiss")
             
-            # Upload to Supabase
-            with open(zip_path, 'rb') as f:
-                response = supabase.storage.from_(bucket_name).upload(file_name, f, {"upsert": "true"})
-                
-            print(f"Vector store uploaded to Supabase: {bucket_name}/{file_name}")
+            # Upload index.pkl
+            pkl_file = os.path.join(local_path, "index.pkl")
+            if os.path.exists(pkl_file):
+                with open(pkl_file, 'rb') as f:
+                    supabase.storage.from_(bucket_name).upload(
+                        f"{file_prefix}_index.pkl", 
+                        f, 
+                        {"upsert": "true"}
+                    )
+                print(f"Uploaded: {file_prefix}_index.pkl")
+            
+            print(f"Vector store uploaded to Supabase bucket: {bucket_name}")
             return True
             
     except Exception as e:
@@ -35,26 +44,34 @@ def save_vector_store_to_supabase(vector_store, supabase, bucket_name, file_name
         st.error(f"Error uploading to Supabase: {e}")
         return False
 
-def load_vector_store_from_supabase(supabase, bucket_name, file_name):
-    """Load vector store from Supabase storage."""
+def load_vector_store_from_supabase(supabase, bucket_name, file_prefix="vector_store"):
+    """Load vector store from Supabase storage from separate files."""
     try:
-        # Download from Supabase
-        response = supabase.storage.from_(bucket_name).download(file_name)
-        
-        if not response:
-            print("Vector store file not found in Supabase.")
-            return None
-        
         with tempfile.TemporaryDirectory() as temp_dir:
-            # Save downloaded zip file
-            zip_path = os.path.join(temp_dir, "vector_store.zip")
-            with open(zip_path, 'wb') as f:
-                f.write(response)
+            local_path = os.path.join(temp_dir, "vector_store")
+            os.makedirs(local_path, exist_ok=True)
+            
+            # Download index.faiss
+            try:
+                faiss_response = supabase.storage.from_(bucket_name).download(f"{file_prefix}_index.faiss")
+                faiss_file = os.path.join(local_path, "index.faiss")
+                with open(faiss_file, 'wb') as f:
+                    f.write(faiss_response)
+                print(f"Downloaded: {file_prefix}_index.faiss")
+            except Exception as e:
+                print(f"Error downloading index.faiss: {e}")
+                return None
             
-            # Extract zip file
-            extract_path = os.path.join(temp_dir, "vector_store")
-            with zipfile.ZipFile(zip_path, 'r') as zipf:
-                zipf.extractall(extract_path)
+            # Download index.pkl
+            try:
+                pkl_response = supabase.storage.from_(bucket_name).download(f"{file_prefix}_index.pkl")
+                pkl_file = os.path.join(local_path, "index.pkl")
+                with open(pkl_file, 'wb') as f:
+                    f.write(pkl_response)
+                print(f"Downloaded: {file_prefix}_index.pkl")
+            except Exception as e:
+                print(f"Error downloading index.pkl: {e}")
+                return None
             
             # Load vector store
             embeddings = HuggingFaceEmbeddings(
@@ -64,12 +81,12 @@ def load_vector_store_from_supabase(supabase, bucket_name, file_name):
             )
             
             vector_store = FAISS.load_local(
-                extract_path, 
+                local_path, 
                 embeddings, 
                 allow_dangerous_deserialization=True
             )
             
-            print(f"Vector store loaded from Supabase: {bucket_name}/{file_name}")
+            print(f"Vector store loaded from Supabase bucket: {bucket_name}")
             return vector_store
             
     except Exception as e: