adding golden dataset

app.py

@@ -14,6 +14,10 @@ import os
 from ragas import evaluate
 from ragas.metrics import answer_relevancy
 from langchain_core.documents import Document
+import json
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
 
 
 load_dotenv()
@@ -21,6 +25,7 @@ load_dotenv()
 # Load OpenAI Model
 llm = ChatOpenAI(model="gpt-4o-mini")
 qd_api_key = os.getenv("QDRANT_CLOUD_API_KEY")
+EVALUATION_MODE = os.getenv("EVALUATION_MODE", "false").lower() == "false"
 
 
 embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
@@ -181,17 +186,19 @@ format_prompt = ChatPromptTemplate.from_template(ot_formatted_prompt)
 def research_node(state) -> dict:
     question = state["messages"][-1].content
 
-    # ✅ Convert the text question to an embedding using OpenAI Embeddings
+    # Convert the text question to an embedding using OpenAI Embeddings
     query_vector = embedding_model.embed_query(question)
 
-    # ✅ Query Qdrant with the vector
-    relevant_docs = search(query_vector=query_vector, top_k=1)
+    # Query Qdrant with the vector
+    relevant_docs = search(query_vector=query_vector, top_k=1)    
 
-    # ✅ Evaluate retrieved documents using RAGAS
-    relevance_score = evaluate_retrieved_docs(question, relevant_docs)
+    if EVALUATION_MODE:
+        # Evaluate retrieved documents using RAGAS
+        relevance_score = evaluate_retrieved_docs(question, relevant_docs)        
+        print(f"📊 [Evaluation Mode] RAGAS Score: {relevance_score}")
 
-    if relevance_score > 0.5:  # Threshold for good retrieval quality
-        # ✅ Found relevant document → Summarize it
+    if relevant_docs[0]['score'] > 0.5:  # Threshold for good retrieval quality this will be the cosine similarity score
+        # Found relevant document → Summarize it
         document_name = relevant_docs[0]["metadata"].get("document_name", "No source available.")
         document_text = get_document_by_name(document_name)
 
@@ -201,17 +208,50 @@ def research_node(state) -> dict:
         return {**state, "messages": state["messages"] + [HumanMessage(content=response.content)], "_next": "post_processing"}
     
     else:
-        # ✅ No relevant document or low RAGAS score → Query LLM directly
-        print("⚠️ RAGAS score too low, defaulting to LLM.")
+        # No relevant document        
         messages = rag_prompt.format_messages(question=question, context="No relevant documents found.")
         response = llm.invoke(messages)
 
         return {**state, "messages": state["messages"] + [HumanMessage(content=response.content)], "_next": "post_processing"}
 
+def compare_text_similarity(text1, text2):
+    """Compute cosine similarity between two texts using embeddings."""
+    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+    
+    emb1 = np.array(embeddings.embed_query(text1)).reshape(1, -1)
+    emb2 = np.array(embeddings.embed_query(text2)).reshape(1, -1)
+
+    return cosine_similarity(emb1, emb2)[0][0]  # Return similarity score
+
+def evaluate_against_golden_set(question, model_answer):
+    """Compare model-generated answers against the golden dataset."""
+    with open("testingset.json", "r", encoding="utf-8") as f:
+        golden_data = json.load(f)
+
+    # Find the corresponding question in the dataset
+    for entry in golden_data:
+        if entry["question"].strip() == question.strip():
+            expected_answer = entry["expected_answer"]
+            break
+    else:
+        print("⚠️ Question not found in the Golden Data Set.")
+        return None
+
+    # Evaluate similarity (simple text match, or use embedding similarity)
+    similarity_score = compare_text_similarity(model_answer, expected_answer)
+    
+    print(f"📊 [Evaluation] Model vs. Expected Score: {similarity_score:.2f}")
+    return similarity_score
 
 # **Post-Processing Node: Formats response using `ot_formatted_prompt`**
 def post_processing_node(state) -> dict:
     response_text = state["messages"][-1].content
+    
+    # Evaluate the model against the golden dataset
+    if EVALUATION_MODE:
+        question = state["messages"][0].content
+        evaluate_against_golden_set(question, response_text)
+
     messages = format_prompt.format_messages(context=response_text)
     response = llm.invoke(messages)
 

data/testingset.json

@@ -0,0 +1,17 @@
+[
+    {
+        "question": "What are the best exercises for tennis elbow?",
+        "contexts": [
+            "Tennis elbow graded exercise protocol...",
+            "Manual therapy review..."
+        ],
+        "expected_answer": "Eccentric exercises and stretching reduce symptoms."
+    },
+    {
+        "question": "What is the role of manual therapy in treating elbow injuries?",
+        "contexts": [
+            "Research shows that manual therapy combined with exercise is beneficial."
+        ],
+        "expected_answer": "Manual therapy improves range of motion and reduces pain."
+    }
+]

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "nltk>=3.9.1",
     "qdrant-client>=1.13.2",
     "ragas>=0.2.13",
+    "scikit-learn>=1.6.1",
     "unstructured>=0.14.8",
     "websockets>=15.0",
 ]

uv.lock

@@ -1271,6 +1271,7 @@ dependencies = [
     { name = "nltk" },
     { name = "qdrant-client" },
     { name = "ragas" },
+    { name = "scikit-learn" },
     { name = "unstructured" },
     { name = "websockets" },
 ]
@@ -1286,6 +1287,7 @@ requires-dist = [
     { name = "nltk", specifier = ">=3.9.1" },
     { name = "qdrant-client", specifier = ">=1.13.2" },
     { name = "ragas", specifier = ">=0.2.13" },
+    { name = "scikit-learn", specifier = ">=1.6.1" },
     { name = "unstructured", specifier = ">=0.14.8" },
     { name = "websockets", specifier = ">=15.0" },
 ]
@@ -1716,6 +1718,58 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481 },
 ]
 
+[[package]]
+name = "scikit-learn"
+version = "1.6.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "joblib" },
+    { name = "numpy" },
+    { name = "scipy" },
+    { name = "threadpoolctl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9e/a5/4ae3b3a0755f7b35a280ac90b28817d1f380318973cff14075ab41ef50d9/scikit_learn-1.6.1.tar.gz", hash = "sha256:b4fc2525eca2c69a59260f583c56a7557c6ccdf8deafdba6e060f94c1c59738e", size = 7068312 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/59/8eb1872ca87009bdcdb7f3cdc679ad557b992c12f4b61f9250659e592c63/scikit_learn-1.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ffa1e9e25b3d93990e74a4be2c2fc61ee5af85811562f1288d5d055880c4322", size = 12010001 },
+    { url = "https://files.pythonhosted.org/packages/9d/05/f2fc4effc5b32e525408524c982c468c29d22f828834f0625c5ef3d601be/scikit_learn-1.6.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:dc5cf3d68c5a20ad6d571584c0750ec641cc46aeef1c1507be51300e6003a7e1", size = 11096360 },
+    { url = "https://files.pythonhosted.org/packages/c8/e4/4195d52cf4f113573fb8ebc44ed5a81bd511a92c0228889125fac2f4c3d1/scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c06beb2e839ecc641366000ca84f3cf6fa9faa1777e29cf0c04be6e4d096a348", size = 12209004 },
+    { url = "https://files.pythonhosted.org/packages/94/be/47e16cdd1e7fcf97d95b3cb08bde1abb13e627861af427a3651fcb80b517/scikit_learn-1.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8ca8cb270fee8f1f76fa9bfd5c3507d60c6438bbee5687f81042e2bb98e5a97", size = 13171776 },
+    { url = "https://files.pythonhosted.org/packages/34/b0/ca92b90859070a1487827dbc672f998da95ce83edce1270fc23f96f1f61a/scikit_learn-1.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:7a1c43c8ec9fde528d664d947dc4c0789be4077a3647f232869f41d9bf50e0fb", size = 11071865 },
+    { url = "https://files.pythonhosted.org/packages/12/ae/993b0fb24a356e71e9a894e42b8a9eec528d4c70217353a1cd7a48bc25d4/scikit_learn-1.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a17c1dea1d56dcda2fac315712f3651a1fea86565b64b48fa1bc090249cbf236", size = 11955804 },
+    { url = "https://files.pythonhosted.org/packages/d6/54/32fa2ee591af44507eac86406fa6bba968d1eb22831494470d0a2e4a1eb1/scikit_learn-1.6.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a7aa5f9908f0f28f4edaa6963c0a6183f1911e63a69aa03782f0d924c830a35", size = 11100530 },
+    { url = "https://files.pythonhosted.org/packages/3f/58/55856da1adec655bdce77b502e94a267bf40a8c0b89f8622837f89503b5a/scikit_learn-1.6.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0650e730afb87402baa88afbf31c07b84c98272622aaba002559b614600ca691", size = 12433852 },
+    { url = "https://files.pythonhosted.org/packages/ff/4f/c83853af13901a574f8f13b645467285a48940f185b690936bb700a50863/scikit_learn-1.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:3f59fe08dc03ea158605170eb52b22a105f238a5d512c4470ddeca71feae8e5f", size = 11337256 },
+]
+
+[[package]]
+name = "scipy"
+version = "1.15.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b7/b9/31ba9cd990e626574baf93fbc1ac61cf9ed54faafd04c479117517661637/scipy-1.15.2.tar.gz", hash = "sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec", size = 59417316 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/53/40/09319f6e0f276ea2754196185f95cd191cb852288440ce035d5c3a931ea2/scipy-1.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:01edfac9f0798ad6b46d9c4c9ca0e0ad23dbf0b1eb70e96adb9fa7f525eff0bf", size = 38717587 },
+    { url = "https://files.pythonhosted.org/packages/fe/c3/2854f40ecd19585d65afaef601e5e1f8dbf6758b2f95b5ea93d38655a2c6/scipy-1.15.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:08b57a9336b8e79b305a143c3655cc5bdbe6d5ece3378578888d2afbb51c4e37", size = 30100266 },
+    { url = "https://files.pythonhosted.org/packages/dd/b1/f9fe6e3c828cb5930b5fe74cb479de5f3d66d682fa8adb77249acaf545b8/scipy-1.15.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:54c462098484e7466362a9f1672d20888f724911a74c22ae35b61f9c5919183d", size = 22373768 },
+    { url = "https://files.pythonhosted.org/packages/15/9d/a60db8c795700414c3f681908a2b911e031e024d93214f2d23c6dae174ab/scipy-1.15.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:cf72ff559a53a6a6d77bd8eefd12a17995ffa44ad86c77a5df96f533d4e6c6bb", size = 25154719 },
+    { url = "https://files.pythonhosted.org/packages/37/3b/9bda92a85cd93f19f9ed90ade84aa1e51657e29988317fabdd44544f1dd4/scipy-1.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9de9d1416b3d9e7df9923ab23cd2fe714244af10b763975bea9e4f2e81cebd27", size = 35163195 },
+    { url = "https://files.pythonhosted.org/packages/03/5a/fc34bf1aa14dc7c0e701691fa8685f3faec80e57d816615e3625f28feb43/scipy-1.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb530e4794fc8ea76a4a21ccb67dea33e5e0e60f07fc38a49e821e1eae3b71a0", size = 37255404 },
+    { url = "https://files.pythonhosted.org/packages/4a/71/472eac45440cee134c8a180dbe4c01b3ec247e0338b7c759e6cd71f199a7/scipy-1.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5ea7ed46d437fc52350b028b1d44e002646e28f3e8ddc714011aaf87330f2f32", size = 36860011 },
+    { url = "https://files.pythonhosted.org/packages/01/b3/21f890f4f42daf20e4d3aaa18182dddb9192771cd47445aaae2e318f6738/scipy-1.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11e7ad32cf184b74380f43d3c0a706f49358b904fa7d5345f16ddf993609184d", size = 39657406 },
+    { url = "https://files.pythonhosted.org/packages/0d/76/77cf2ac1f2a9cc00c073d49e1e16244e389dd88e2490c91d84e1e3e4d126/scipy-1.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:a5080a79dfb9b78b768cebf3c9dcbc7b665c5875793569f48bf0e2b1d7f68f6f", size = 40961243 },
+    { url = "https://files.pythonhosted.org/packages/4c/4b/a57f8ddcf48e129e6054fa9899a2a86d1fc6b07a0e15c7eebff7ca94533f/scipy-1.15.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:447ce30cee6a9d5d1379087c9e474628dab3db4a67484be1b7dc3196bfb2fac9", size = 38870286 },
+    { url = "https://files.pythonhosted.org/packages/0c/43/c304d69a56c91ad5f188c0714f6a97b9c1fed93128c691148621274a3a68/scipy-1.15.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c90ebe8aaa4397eaefa8455a8182b164a6cc1d59ad53f79943f266d99f68687f", size = 30141634 },
+    { url = "https://files.pythonhosted.org/packages/44/1a/6c21b45d2548eb73be9b9bff421aaaa7e85e22c1f9b3bc44b23485dfce0a/scipy-1.15.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:def751dd08243934c884a3221156d63e15234a3155cf25978b0a668409d45eb6", size = 22415179 },
+    { url = "https://files.pythonhosted.org/packages/74/4b/aefac4bba80ef815b64f55da06f62f92be5d03b467f2ce3668071799429a/scipy-1.15.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:302093e7dfb120e55515936cb55618ee0b895f8bcaf18ff81eca086c17bd80af", size = 25126412 },
+    { url = "https://files.pythonhosted.org/packages/b1/53/1cbb148e6e8f1660aacd9f0a9dfa2b05e9ff1cb54b4386fe868477972ac2/scipy-1.15.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd5b77413e1855351cdde594eca99c1f4a588c2d63711388b6a1f1c01f62274", size = 34952867 },
+    { url = "https://files.pythonhosted.org/packages/2c/23/e0eb7f31a9c13cf2dca083828b97992dd22f8184c6ce4fec5deec0c81fcf/scipy-1.15.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d0194c37037707b2afa7a2f2a924cf7bac3dc292d51b6a925e5fcb89bc5c776", size = 36890009 },
+    { url = "https://files.pythonhosted.org/packages/03/f3/e699e19cabe96bbac5189c04aaa970718f0105cff03d458dc5e2b6bd1e8c/scipy-1.15.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:bae43364d600fdc3ac327db99659dcb79e6e7ecd279a75fe1266669d9a652828", size = 36545159 },
+    { url = "https://files.pythonhosted.org/packages/af/f5/ab3838e56fe5cc22383d6fcf2336e48c8fe33e944b9037fbf6cbdf5a11f8/scipy-1.15.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f031846580d9acccd0044efd1a90e6f4df3a6e12b4b6bd694a7bc03a89892b28", size = 39136566 },
+    { url = "https://files.pythonhosted.org/packages/0a/c8/b3f566db71461cabd4b2d5b39bcc24a7e1c119535c8361f81426be39bb47/scipy-1.15.2-cp313-cp313t-win_amd64.whl", hash = "sha256:fe8a9eb875d430d81755472c5ba75e84acc980e4a8f6204d402849234d3017db", size = 40477705 },
+]
+
 [[package]]
 name = "setuptools"
 version = "75.8.0"
@@ -1821,6 +1875,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b6/cb/b86984bed139586d01532a587464b5805f12e397594f19f931c4c2fbfa61/tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539", size = 28169 },
 ]
 
+[[package]]
+name = "threadpoolctl"
+version = "3.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bd/55/b5148dcbf72f5cde221f8bfe3b6a540da7aa1842f6b491ad979a6c8b84af/threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107", size = 41936 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4b/2c/ffbf7a134b9ab11a67b0cf0726453cedd9c5043a4fe7a35d1cefa9a1bcfb/threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467", size = 18414 },
+]
+
 [[package]]
 name = "tiktoken"
 version = "0.9.0"