refactor: Add contextual compression retriever to ensemble retriever

libs/retrievers.py

@@ -1,5 +1,7 @@
 # retrievers.py
 import pickle
+from langchain_cohere import CohereRerank
+from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
 from langchain_community.vectorstores import FAISS
 from langchain.retrievers import EnsembleRetriever
 
@@ -26,8 +28,15 @@ def load_retrievers(embeddings):
 
     bm25_retriever = load_bm25_retriever().with_config(run_name="bm25")
 
-    return EnsembleRetriever(
+    ensemble_retriever = EnsembleRetriever(
         retrievers=[bm25_retriever, faiss_retriever],
         weights=[0.7, 0.3],
         search_type="mmr",
     )
+
+    compressor = CohereRerank(model="rerank-multilingual-v3.0", top_n=5)
+    compression_retriever = ContextualCompressionRetriever(
+        base_compressor=compressor,
+        base_retriever=ensemble_retriever,
+    )
+    return compression_retriever