Implements clear index but tfidf still struggles

.streamlit/config.toml

@@ -1,5 +1,5 @@
 [theme]
-primaryColor="#ffbf00"
+primaryColor="#e5ab00"
 backgroundColor="#0e1117"
 secondaryBackgroundColor="#282929"
 textColor = "#ffffff"

app.py

@@ -29,7 +29,7 @@ def run_demo():
     with navigation:
 
         selected_page = option_menu(
-            menu_title="Navigation",
+            menu_title=None,
             options=list(pages.keys()),
             icons=[f[1] for f in pages.values()],
             menu_icon="cast",

core/pipelines.py

@@ -39,10 +39,7 @@ def keyword_search(index="documents", split_word_length=100):
     index_pipeline = Pipeline()
     index_pipeline.add_node(processor, name="Preprocessor", inputs=["File"])
     index_pipeline.add_node(
-        keyword_retriever, name="TfidfRetriever", inputs=["Preprocessor"]
-    )
-    index_pipeline.add_node(
-        document_store, name="DocumentStore", inputs=["TfidfRetriever"]
+        document_store, name="DocumentStore", inputs=["Preprocessor"]
     )
 
     return search_pipeline, index_pipeline

core/search_index.py

@@ -1,4 +1,5 @@
 from haystack.schema import Document
+from haystack.document_stores import BaseDocumentStore
 import uuid
 
 
@@ -17,8 +18,14 @@ def format_docs(documents):
     return db_docs, [doc.meta["id"] for doc in db_docs]
 
 
-def index(documents, pipeline):
+def index(documents, pipeline, clear_index=True):
     documents, doc_ids = format_docs(documents)
+    if clear_index:
+        document_stores = pipeline.get_nodes_by_class(
+            class_type=BaseDocumentStore
+        )
+        for docstore in document_stores:
+            docstore.delete_index(docstore.index)
     pipeline.run(documents=documents)
     return doc_ids
 

interface/pages.py

@@ -78,6 +78,8 @@ def page_index(container):
             default_index=0,
             orientation="horizontal",
         )
+        
+        clear_index = st.sidebar.checkbox('Clear Index', True)
 
         corpus = input_funcs[selected_input][0](container)
 
@@ -87,6 +89,7 @@ def page_index(container):
                 index_results = index(
                     corpus,
                     st.session_state["pipeline"]["index_pipeline"],
+                    clear_index
                 )
             if index_results:
                 st.write(index_results)