to create RAGAs result with triad of metrics

.gitignore

@@ -3,10 +3,10 @@
 .streamlit/
 results/
 
-*.sqlite
 data/
 
 notebooks/test_model
 screenshot_questions/
 
-# ux/
+# ux/
+# *.sqlite

archive/dependencies/requirements_llama-index==0.9.24.txt

@@ -0,0 +1,259 @@
+aiohttp==3.9.1
+aiosignal==1.3.1
+alembic==1.13.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+appnope==0.1.3
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asgiref==3.7.2
+asttokens==2.4.1
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.2.0
+Babel==2.14.0
+backoff==2.2.1
+bcrypt==4.1.2
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blinker==1.7.0
+bs4==0.0.2
+build==1.0.3
+cachetools==5.3.2
+certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.22
+click==8.1.7
+coloredlogs==15.0.1
+comm==0.2.0
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.3
+debugpy==1.8.0
+decorator==5.1.1
+defusedxml==0.7.1
+Deprecated==1.2.14
+dill==0.3.7
+dirtyjson==1.0.8
+distro==1.9.0
+entrypoints==0.4
+exceptiongroup==1.2.0
+executing==2.0.1
+Faker==22.0.0
+fastapi==0.109.0
+fastjsonschema==2.19.1
+favicon==0.7.0
+filelock==3.13.1
+flatbuffers==23.5.26
+fonttools==4.47.0
+fqdn==1.5.1
+frozendict==2.4.0
+frozenlist==1.4.1
+fsspec==2023.12.2
+gitdb==4.0.11
+GitPython==3.1.40
+google-auth==2.27.0
+googleapis-common-protos==1.62.0
+greenlet==3.0.3
+grpcio==1.60.0
+h11==0.14.0
+htbuilder==0.6.2
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.26.0
+huggingface-hub==0.20.1
+humanfriendly==10.0
+humanize==4.9.0
+idna==3.6
+importlib-metadata==6.11.0
+importlib-resources==6.1.1
+ipykernel==6.28.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.14
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.20.0
+jsonschema-specifications==2023.12.1
+jupyter==1.0.0
+jupyter-console==6.6.3
+jupyter-events==0.9.0
+jupyter-lsp==2.2.1
+jupyter_client==8.6.0
+jupyter_core==5.6.1
+jupyter_server==2.12.1
+jupyter_server_terminals==0.5.1
+jupyterlab==4.0.10
+jupyterlab-widgets==3.0.9
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.25.2
+kiwisolver==1.4.5
+kubernetes==29.0.0
+langchain==0.0.354
+langchain-community==0.0.8
+langchain-core==0.1.23
+langsmith==0.0.87
+llama-index==0.9.24
+lxml==5.1.0
+Mako==1.3.0
+Markdown==3.5.1
+markdown-it-py==3.0.0
+markdownlit==0.0.7
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.2
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+merkle-json==1.0.0
+millify==0.1.1
+mistune==3.0.2
+mmh3==4.1.0
+monotonic==1.6
+more-itertools==10.1.0
+mpmath==1.3.0
+multidict==6.0.4
+munch==4.0.0
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.14.0
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+nltk==3.8.1
+notebook==7.0.6
+notebook_shim==0.2.3
+numpy==1.26.2
+oauthlib==3.2.2
+onnxruntime==1.17.0
+openai==1.6.1
+opentelemetry-api==1.22.0
+opentelemetry-exporter-otlp-proto-common==1.22.0
+opentelemetry-exporter-otlp-proto-grpc==1.22.0
+opentelemetry-instrumentation==0.43b0
+opentelemetry-instrumentation-asgi==0.43b0
+opentelemetry-instrumentation-fastapi==0.43b0
+opentelemetry-proto==1.22.0
+opentelemetry-sdk==1.22.0
+opentelemetry-semantic-conventions==0.43b0
+opentelemetry-util-http==0.43b0
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.4
+pandocfilters==1.5.0
+parso==0.8.3
+pexpect==4.9.0
+pillow==10.2.0
+platformdirs==4.1.0
+posthog==3.3.3
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.25.1
+psutil==5.9.7
+ptyprocess==0.7.0
+pulsar-client==3.4.0
+pure-eval==0.2.2
+pyarrow==14.0.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pycparser==2.21
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydeck==0.8.1b0
+Pygments==2.17.2
+pymdown-extensions==10.7
+PyMuPDF==1.23.22
+PyMuPDFb==1.23.22
+pyparsing==3.1.1
+pypdf==4.0.1
+PyPika==0.48.9
+pyproject_hooks==1.0.0
+python-dateutil==2.8.2
+python-decouple==3.8
+python-dotenv==1.0.0
+python-json-logger==2.0.7
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==25.1.2
+qtconsole==5.5.1
+QtPy==2.4.1
+referencing==0.32.0
+regex==2023.12.25
+requests==2.31.0
+requests-oauthlib==1.3.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.0
+rpds-py==0.16.2
+rsa==4.9
+safetensors==0.4.1
+scikit-learn==1.4.0
+scipy==1.12.0
+Send2Trash==1.8.2
+sentence-transformers==2.3.0
+sentencepiece==0.1.99
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.24
+st-annotated-text==4.0.1
+stack-data==0.6.3
+starlette==0.35.1
+streamlit==1.31.1
+streamlit-aggrid==0.3.4.post3
+streamlit-camera-input-live==0.2.0
+streamlit-card==1.0.0
+streamlit-embedcode==0.1.2
+streamlit-extras==0.3.6
+streamlit-faker==0.0.3
+streamlit-feedback==0.1.3
+streamlit-image-coordinates==0.1.6
+streamlit-keyup==0.2.2
+streamlit-toggle-switch==1.0.2
+streamlit-vertical-slider==2.5.5
+sympy==1.12
+tenacity==8.2.3
+terminado==0.18.0
+threadpoolctl==3.2.0
+tiktoken==0.5.2
+tinycss2==1.2.1
+tokenizers==0.15.2
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+torch==2.1.2
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.14.0
+transformers==4.37.2
+trulens==0.13.4
+trulens-eval==0.20.0
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+tzlocal==5.2
+uri-template==1.3.0
+urllib3==2.1.0
+uvicorn==0.27.0
+uvloop==0.19.0
+validators==0.22.0
+watchfiles==0.21.0
+wcwidth==0.2.12
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+websockets==12.0
+widgetsnbextension==4.0.9
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.17.0

archive/model_evaluation/main_new.py

@@ -0,0 +1,180 @@
+import utils
+import os
+
+import numpy as np
+import nest_asyncio
+import openai
+import chromadb
+
+from llama_index.legacy import (
+                VectorStoreIndex,
+                SimpleDirectoryReader
+)                               
+from llama_index.core import (               
+                StorageContext,
+                Document,
+                Settings
+)
+from llama_index.vector_stores.chroma.base import ChromaVectorStore
+from llama_index.llms.openai import OpenAI
+from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding
+from trulens_eval import Tru
+
+from utils import get_prebuilt_trulens_recorder
+import time
+
+nest_asyncio.apply()
+openai.api_key = utils.get_openai_api_key()
+
+def main():
+    
+    if not os.path.exists("./default.sqlite"):
+        
+        start_time = time.time()
+
+        llm = OpenAI(model="gpt-3.5-turbo-1106", temperature=0.0)
+        fine_tuned_path = "local:./models/fine-tuned-embeddings"
+        
+        Settings.llm = llm
+        Settings.embed_model = fine_tuned_path
+
+        db = chromadb.PersistentClient(path="./models/chroma_db")
+        chroma_collection = db.get_or_create_collection("quickstart")
+
+        # assign chroma as the vector_store to the context
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+        # create your index
+        index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store,
+            storage_context=storage_context
+        )
+        query_engine = index.as_query_engine()
+
+        separator = "\n\n"
+        eval_questions = []
+        with open('./raw_documents/eval_questions.txt', 'r') as file:
+            content = file.read()
+
+        for question in content.split(separator):
+            print(question)
+            print(separator)
+            eval_questions.append(question.strip())
+
+        response = query_engine.query(eval_questions[0])
+        print(str(response))
+
+        tru = Tru(database_file="./models/trulens_eval.sqlite")
+        tru_recorder = get_prebuilt_trulens_recorder(query_engine,
+                                                     app_id="Direct Query Engine")
+        
+        print("Sending each question to llm ..")
+        with tru_recorder as recording:
+            for question in eval_questions:
+                response = query_engine.query(question)
+
+        records, feedback = tru.get_records_and_feedback(app_ids=[])
+
+        os.makedirs("./results", exist_ok=True)
+        records.to_csv("./results/records.csv", index=False)
+
+        print(tru.db.engine.url.render_as_string(hide_password=False))
+        
+        end_time = time.time()
+        time_spent_mins = (end_time - start_time) / 60
+        with open("./results/time_cost.txt", "w") as fp:
+            fp.write(f"Takes {int(time_spent_mins)} mins to create llm evaluation.")
+
+if __name__ == "__main__":
+
+    # main()
+    if False:
+        start_time = time.time()
+
+        llm = OpenAI(model="gpt-3.5-turbo-1106", temperature=0.0)
+        fine_tuned_path = "local:./models/fine-tuned-embeddings"
+        
+        Settings.llm = llm
+        Settings.embed_model = fine_tuned_path
+
+        db = chromadb.PersistentClient(path="./models/chroma_db")
+        chroma_collection = db.get_or_create_collection("quickstart")
+
+        # assign chroma as the vector_store to the context
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+        # create your index
+        index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store,
+            storage_context=storage_context
+        )
+        query_engine = index.as_query_engine()
+
+        separator = "\n\n"
+        eval_questions = []
+        with open('./raw_documents/eval_questions.txt', 'r') as file:
+            content = file.read()
+
+        for question in content.split(separator):
+            print(question)
+            print(separator)
+            eval_questions.append(question.strip())
+
+        response = query_engine.query(eval_questions[0])
+        print(str(response))
+
+
+
+    from trulens_eval import Tru
+    tru = Tru()
+
+    documents = SimpleDirectoryReader(
+        input_files=["./raw_documents/qna.txt"]
+    ).load_data()
+    index = VectorStoreIndex.from_documents(documents)
+
+    query_engine = index.as_query_engine()
+    response = query_engine.query("Which is not a government healthcare philosophy?")
+    print(response)
+
+    from trulens_eval.feedback.provider.openai import OpenAI
+    openai = OpenAI()
+
+    # select context to be used in feedback. the location of context is app specific.
+    from trulens_eval.app import App
+    context = App.select_context(query_engine)
+
+    from trulens_eval import Feedback
+
+    # Define a groundedness feedback function
+    from trulens_eval.feedback import Groundedness
+    grounded = Groundedness(groundedness_provider=OpenAI())
+    f_groundedness = (
+        Feedback(grounded.groundedness_measure_with_cot_reasons)
+        .on(context.collect()) # collect context chunks into a list
+        .on_output()
+        .aggregate(grounded.grounded_statements_aggregator)
+    )
+
+    # Question/answer relevance between overall question and answer.
+    f_qa_relevance = Feedback(openai.relevance).on_input_output()
+
+    # Question/statement relevance between question and each context chunk.
+    f_qs_relevance = (
+        Feedback(openai.qs_relevance)
+        .on_input()
+        .on(context)
+        .aggregate(np.mean)
+    )
+
+    from trulens_eval import TruLlama
+    tru_query_engine_recorder = TruLlama(query_engine,
+        app_id='LlamaIndex_App1',
+        feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance])
+    
+    if False:
+        # or as context manager
+        with tru_query_engine_recorder as recording:
+            query_engine.query("Which of the following is TRUE on the similarity of Means Testing and Casemix?")

archive/model_evaluation/utils.py

@@ -0,0 +1,160 @@
+import os
+import numpy as np
+from trulens_eval import (
+    Feedback,
+    TruLlama,
+    OpenAI
+)
+
+from trulens_eval.feedback import Groundedness
+import nest_asyncio
+
+from llama_index.core import ServiceContext, VectorStoreIndex, StorageContext
+from llama_index.core import load_index_from_storage
+from llama_index.core.node_parser import HierarchicalNodeParser
+from llama_index.core.node_parser import get_leaf_nodes
+
+from llama_index.packs.auto_merging_retriever.base import AutoMergingRetrieverPack
+
+
+from llama_index.node_parser import SentenceWindowNodeParser
+from llama_index.indices.postprocessor import MetadataReplacementPostProcessor
+from llama_index.indices.postprocessor import SentenceTransformerRerank
+from llama_index.query_engine import RetrieverQueryEngine
+
+
+nest_asyncio.apply()
+openai = OpenAI()
+
+qa_relevance = (
+    Feedback(openai.relevance_with_cot_reasons, name="Answer Relevance")
+    .on_input_output()
+)
+
+qs_relevance = (
+    Feedback(openai.relevance_with_cot_reasons, name = "Context Relevance")
+    .on_input()
+    .on(TruLlama.select_source_nodes().node.text)
+    .aggregate(np.mean)
+)
+
+#grounded = Groundedness(groundedness_provider=openai, summarize_provider=openai)
+grounded = Groundedness(groundedness_provider=openai)
+
+groundedness = (
+    Feedback(grounded.groundedness_measure_with_cot_reasons, name="Groundedness")
+        .on(TruLlama.select_source_nodes().node.text)
+        .on_output()
+        .aggregate(grounded.grounded_statements_aggregator)
+)
+
+feedbacks = [qa_relevance, qs_relevance, groundedness]
+
+def get_openai_api_key():
+    return os.getenv("OPENAI_API_KEY")
+
+def get_trulens_recorder(query_engine, feedbacks, app_id):
+    tru_recorder = TruLlama(
+        query_engine,
+        app_id=app_id,
+        feedbacks=feedbacks
+    )
+    return tru_recorder
+
+def get_prebuilt_trulens_recorder(query_engine, app_id):
+    tru_recorder = TruLlama(
+        query_engine,
+        app_id=app_id,
+        feedbacks=feedbacks
+        )
+    return tru_recorder
+
+def build_sentence_window_index(
+    document, llm, embed_model="local:BAAI/bge-small-en-v1.5", save_dir="sentence_index"
+):
+    # create the sentence window node parser w/ default settings
+    node_parser = SentenceWindowNodeParser.from_defaults(
+        window_size=3,
+        window_metadata_key="window",
+        original_text_metadata_key="original_text",
+    )
+    sentence_context = ServiceContext.from_defaults(
+        llm=llm,
+        embed_model=embed_model,
+        node_parser=node_parser,
+    )
+    if not os.path.exists(save_dir):
+        sentence_index = VectorStoreIndex.from_documents(
+            [document], service_context=sentence_context
+        )
+        sentence_index.storage_context.persist(persist_dir=save_dir)
+    else:
+        sentence_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir),
+            service_context=sentence_context,
+        )
+
+    return sentence_index
+
+def get_sentence_window_query_engine(
+    sentence_index,
+    similarity_top_k=6,
+    rerank_top_n=2,
+):
+    # define postprocessors
+    postproc = MetadataReplacementPostProcessor(target_metadata_key="window")
+    rerank = SentenceTransformerRerank(
+        top_n=rerank_top_n, model="BAAI/bge-reranker-base"
+    )
+
+    sentence_window_engine = sentence_index.as_query_engine(
+        similarity_top_k=similarity_top_k, node_postprocessors=[postproc, rerank]
+    )
+    return sentence_window_engine
+
+def build_automerging_index(
+    documents,
+    llm,
+    embed_model="local:BAAI/bge-small-en-v1.5",
+    save_dir="merging_index",
+    chunk_sizes=None,
+):
+    chunk_sizes = chunk_sizes or [2048, 512, 128]
+    node_parser = HierarchicalNodeParser.from_defaults(chunk_sizes=chunk_sizes)
+    nodes = node_parser.get_nodes_from_documents(documents)
+    leaf_nodes = get_leaf_nodes(nodes)
+    merging_context = ServiceContext.from_defaults(
+        llm=llm,
+        embed_model=embed_model,
+    )
+    storage_context = StorageContext.from_defaults()
+    storage_context.docstore.add_documents(nodes)
+
+    if not os.path.exists(save_dir):
+        automerging_index = VectorStoreIndex(
+            leaf_nodes, storage_context=storage_context, service_context=merging_context
+        )
+        automerging_index.storage_context.persist(persist_dir=save_dir)
+    else:
+        automerging_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir),
+            service_context=merging_context,
+        )
+    return automerging_index
+
+def get_automerging_query_engine(
+    automerging_index,
+    similarity_top_k=12,
+    rerank_top_n=2,
+):
+    base_retriever = automerging_index.as_retriever(similarity_top_k=similarity_top_k)
+    retriever = AutoMergingRetriever(
+        base_retriever, automerging_index.storage_context, verbose=True
+    )
+    rerank = SentenceTransformerRerank(
+        top_n=rerank_top_n, model="BAAI/bge-reranker-base"
+    )
+    auto_merging_engine = RetrieverQueryEngine.from_args(
+        retriever, node_postprocessors=[rerank]
+    )
+    return auto_merging_engine

archive/model_evaluation/utils_new.py

@@ -0,0 +1,95 @@
+import os
+import numpy as np
+from trulens_eval import (
+    Feedback,
+    TruLlama,
+    OpenAI
+)
+
+from trulens_eval.feedback import Groundedness
+import nest_asyncio
+
+from llama_index.core import ServiceContext, VectorStoreIndex, StorageContext
+from llama_index.core import load_index_from_storage
+from llama_index.core.node_parser import HierarchicalNodeParser
+from llama_index.core.node_parser import get_leaf_nodes
+from llama_index.core.query_engine import RetrieverQueryEngine
+
+from llama_index.packs.auto_merging_retriever.base import AutoMergingRetrieverPack
+
+
+nest_asyncio.apply()
+openai = OpenAI()
+
+qa_relevance = (
+    Feedback(openai.relevance_with_cot_reasons, name="Answer Relevance")
+    .on_input_output()
+)
+
+qs_relevance = (
+    Feedback(openai.relevance_with_cot_reasons, name = "Context Relevance")
+    .on_input()
+    .on(TruLlama.select_source_nodes().node.text)
+    .aggregate(np.mean)
+)
+
+#grounded = Groundedness(groundedness_provider=openai, summarize_provider=openai)
+grounded = Groundedness(groundedness_provider=openai)
+
+groundedness = (
+    Feedback(grounded.groundedness_measure_with_cot_reasons, name="Groundedness")
+        .on(TruLlama.select_source_nodes().node.text)
+        .on_output()
+        .aggregate(grounded.grounded_statements_aggregator)
+)
+
+feedbacks = [qa_relevance, qs_relevance, groundedness]
+
+def get_openai_api_key():
+    return os.getenv("OPENAI_API_KEY")
+
+def get_trulens_recorder(query_engine, feedbacks, app_id):
+    tru_recorder = TruLlama(
+        query_engine,
+        app_id=app_id,
+        feedbacks=feedbacks
+    )
+    return tru_recorder
+
+def get_prebuilt_trulens_recorder(query_engine, app_id):
+    tru_recorder = TruLlama(
+        query_engine,
+        app_id=app_id,
+        feedbacks=feedbacks
+        )
+    return tru_recorder
+
+def build_automerging_index(
+    documents,
+    llm,
+    embed_model="local:BAAI/bge-small-en-v1.5",
+    save_dir="merging_index",
+    chunk_sizes=None,
+):
+    chunk_sizes = chunk_sizes or [2048, 512, 128]
+    node_parser = HierarchicalNodeParser.from_defaults(chunk_sizes=chunk_sizes)
+    nodes = node_parser.get_nodes_from_documents(documents)
+    leaf_nodes = get_leaf_nodes(nodes)
+    merging_context = ServiceContext.from_defaults(
+        llm=llm,
+        embed_model=embed_model,
+    )
+    storage_context = StorageContext.from_defaults()
+    storage_context.docstore.add_documents(nodes)
+
+    if not os.path.exists(save_dir):
+        automerging_index = VectorStoreIndex(
+            leaf_nodes, storage_context=storage_context, service_context=merging_context
+        )
+        automerging_index.storage_context.persist(persist_dir=save_dir)
+    else:
+        automerging_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir),
+            service_context=merging_context,
+        )
+    return automerging_index

database/mock_qna_source.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b604288137e94da640f1e5a88900390084eba746508cd7257dbcdba8cbe67f32
-size 2701
+oid sha256:c80d88333c3b9fb2a700d49113d2ba3fef7cc671c11b640168c389bef411bc05
+size 7624

evaluate_model.py

@@ -0,0 +1,83 @@
+import os, time
+import pandas as pd
+from tqdm import tqdm
+
+import chromadb
+import openai
+from llama_index import (
+            SimpleDirectoryReader,
+            StorageContext,
+            Document,
+            VectorStoreIndex,
+            ServiceContext
+)
+
+from llama_index.vector_stores.chroma import ChromaVectorStore
+from llama_index.llms import OpenAI
+from llama_index.embeddings import HuggingFaceEmbedding
+from trulens_eval import Tru
+
+import utils
+from utils import get_prebuilt_trulens_recorder
+
+openai.api_key = utils.get_openai_api_key()
+
+def main():
+    
+    start_time = time.time()
+
+    llm = OpenAI(model="gpt-3.5-turbo-1106", temperature=0.0)
+    fine_tuned_path = "local:./models/fine-tuned-embeddings"
+
+    db = chromadb.PersistentClient(path="./models/chroma_db")
+    chroma_collection = db.get_or_create_collection("quickstart")
+
+    # assign chroma as the vector_store to the context
+    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    service_context = ServiceContext.from_defaults(llm=llm, embed_model=fine_tuned_path)
+
+    print("Loading embeddings from vector store..")
+    index = VectorStoreIndex.from_vector_store(
+        vector_store=vector_store,
+        storage_context=storage_context,
+        service_context=service_context
+    )
+    query_engine = index.as_query_engine()
+    
+    mock_qna_source = pd.read_csv("./database/mock_qna_source.csv")
+    mock_qna_source = mock_qna_source[ mock_qna_source["question"].notnull() ]
+    print("mock_qna_source.shape", mock_qna_source.shape)
+
+    with open("./raw_documents/eval_questions.txt", "r") as fp:
+        questions_content = fp.read()
+        questions_content_ls = questions_content.split("\n\n")
+
+    eval_questions = mock_qna_source["question"].tolist() + questions_content_ls
+    response = query_engine.query(eval_questions[0])
+    print(str(response))
+
+    tru = Tru(database_file="./models/trulens_eval.sqlite")
+    tru_recorder = get_prebuilt_trulens_recorder(query_engine,
+                                                 app_id="Direct Query Engine")
+    
+    print("Sending each question to llm..")
+    with tru_recorder as recording:
+        for question in tqdm(eval_questions):
+            response = query_engine.query(question)
+
+    records, feedback = tru.get_records_and_feedback(app_ids=[])
+
+    os.makedirs("./results", exist_ok=True)
+    records.to_csv("./results/records.csv", index=False)
+
+    print(tru.db.engine.url.render_as_string(hide_password=False))
+    
+    end_time = time.time()
+    time_spent_mins = (end_time - start_time) / 60
+    with open("./results/time_cost.txt", "w") as fp:
+        fp.write(f"Takes {int(time_spent_mins)} mins to create llm evaluation.")
+
+if __name__ == "__main__":
+
+    main()

models/trulens_eval.sqlite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6849488edfa526805c51322b217557de99ac01882a9d2a136a351a68c6b305d5
+size 2936832

notebooks/002_persisted-embedding-model.ipynb

@@ -236,6 +236,7 @@
     "from llama_index.core import StorageContext\n",
     "from llama_index.core import ServiceContext\n",
     "from llama_index.core import Document\n",
+    "from llama_index.core import Settings\n",
     "\n",
     "from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding\n",
     "from llama_index.llms.openai import OpenAI\n",

pages/1_Leaderboard.py

@@ -31,7 +31,7 @@ database_url = None
 
 
 def streamlit_app():
-    tru = Tru(database_url=database_url)
+    tru = Tru(database_file="./models/trulens_eval.sqlite")
     lms = tru.db
 
     # Set the title and subtitle of the app

pages/2_Evaluations.py

@@ -48,7 +48,7 @@ st.runtime.legacy_caching.clear_cache()
 
 add_logo_and_style_overrides()
 
-tru = Tru()
+tru = Tru(database_file="./models/trulens_eval.sqlite")
 lms = tru.db
 
 df_results, feedback_cols = lms.get_records_and_feedback([])
@@ -143,7 +143,7 @@ else:
     else:
         app = apps
 
-    st.experimental_set_query_params(app=app)
+    st.query_params["app"] = app
 
     options = st.multiselect("Filter Applications", apps, default=app)
 

pages/3_app.py

@@ -4,13 +4,11 @@ import os
 try:
     raw_docs_files = ", ".join(os.listdir("./raw_documents"))
     curr_directory_files = ", ".join(os.listdir("."))
+    with open("./raw_documents/eval_answers.txt", "r") as fp:
+        eval_answers = fp.read()
 
-    with open("./results/time_cost.txt", "r") as fp:
-        time_cost_str = fp.read()
-
-    system_update = raw_docs_files + "\n\n" + curr_directory_files + "\n\n" + time_cost_str
-
+    system_update = raw_docs_files + "\n\n" + curr_directory_files + "\n\n" + eval_answers
 except:
     system_update = "NA"
 
-st.write(f"Hello World! File list: {system_update}")
+st.write(f"Hello World! Info about the app: {system_update}")

qna_prompting.py

@@ -22,7 +22,11 @@ class QnA_Model(BaseModel):
                            description=(
                                 "which chapter to extract, the format of this function argumet"
                                 "is with `Chapter_` as prefix concatenated with chapter number"
-                                "in integer. For example, `Chapter_2`, `Chapter_10`.")
+                                "in integer. For example, `Chapter_2`, `Chapter_10`."
+                                "if no chapter number specified or user requested for random question"
+                                "or user has no preference over which chapter of textbook to be tested"
+                                "return `Chapter_0`"
+                                )
                           )
 
 def get_qna_question(chapter_n: str) -> str:
@@ -37,11 +41,12 @@ def get_qna_question(chapter_n: str) -> str:
     """
     con = sqlite3.connect(db_path)
     cur = con.cursor()
-
-    sql_string = f"""SELECT id, question, option_1, option_2, option_3, option_4, correct_answer
-                     FROM qna_tbl
-                     WHERE chapter='{chapter_n}'
-                  """
+    
+    filter_clause = "" if chapter_n == "Chapter_0" else f"WHERE chapter='{chapter_n}'"
+    sql_string = """SELECT id, question, option_1, option_2, option_3, option_4, correct_answer
+                    FROM qna_tbl
+                 """ + filter_clause
+    
     res = cur.execute(sql_string)
     result = res.fetchone()
 

raw_documents/eval_answers.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d6f42a68ccf96496a6dcd89016e53ebb1add84c42ecef1fffe08e211037c4df
-size 332
+oid sha256:0ac533f41fb123fe9281d27f2a3166e997f09c37178d12f5cbbea1fedeb5026b
+size 1458

raw_documents/eval_questions.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:931b5c74d5696e5efb242c7d968765734a621d881642a1b16dbd1d004fd2900e
-size 1473
+oid sha256:7bae3f2ac0cf2fdb2f58de8ecaa8d63014a4f84aa8a839dc7ff0d4ae8eb0eb22
+size 1126

raw_documents/qna.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96f148c23c11fe6df506f5286d2c90143b274ce2705501deaeac47fa63863825
-size 2134
+oid sha256:59cc1b620ccad1393fc4311e91e538774ac76149a42bd2391af7c855895d80bc
+size 56746

requirements.txt

@@ -99,8 +99,8 @@ kiwisolver==1.4.5
 kubernetes==29.0.0
 langchain==0.0.354
 langchain-community==0.0.8
-langchain-core==0.1.5
-langsmith==0.0.77
+langchain-core==0.1.23
+langsmith==0.0.87
 llama-index==0.10.1
 llama-index-agent-openai==0.1.1
 llama-index-core==0.10.1
@@ -109,6 +109,7 @@ llama-index-embeddings-openai==0.1.1
 llama-index-legacy==0.9.48
 llama-index-llms-openai==0.1.1
 llama-index-multi-modal-llms-openai==0.1.1
+llama-index-packs-auto-merging-retriever==0.1.2
 llama-index-program-openai==0.1.1
 llama-index-question-gen-openai==0.1.1
 llama-index-readers-file==0.1.2
@@ -218,7 +219,7 @@ SQLAlchemy==2.0.24
 st-annotated-text==4.0.1
 stack-data==0.6.3
 starlette==0.35.1
-streamlit==1.29.0
+streamlit==1.31.1
 streamlit-aggrid==0.3.4.post3
 streamlit-camera-input-live==0.2.0
 streamlit-card==1.0.0
@@ -246,7 +247,7 @@ tqdm==4.66.1
 traitlets==5.14.0
 transformers==4.37.2
 trulens==0.13.4
-trulens-eval==0.20.0
+trulens-eval==0.22.2
 typer==0.9.0
 types-python-dateutil==2.8.19.14
 typing-inspect==0.9.0

streamlit_app.py

@@ -71,7 +71,7 @@ with st.sidebar:
 
     st.subheader("Models and parameters")
     selected_model = st.sidebar.selectbox("Choose an OpenAI model", 
-                                          ["gpt-3.5-turbo-1106", "gpt-4-1106-preview"], 
+                                          ["gpt-3.5-turbo-0125", "gpt-4-0125-preview"], 
                                            key="selected_model")
     temperature = st.sidebar.slider("temperature", min_value=0.0, max_value=2.0, 
                                     value=0.0, step=0.01)

utils.py

@@ -5,27 +5,18 @@ from trulens_eval import (
     TruLlama,
     OpenAI
 )
-
 from trulens_eval.feedback import Groundedness
-import nest_asyncio
 
 from llama_index import ServiceContext, VectorStoreIndex, StorageContext
-from llama_index.node_parser import SentenceWindowNodeParser
-from llama_index.indices.postprocessor import MetadataReplacementPostProcessor
-from llama_index.indices.postprocessor import SentenceTransformerRerank
 from llama_index import load_index_from_storage
-
 from llama_index.node_parser import HierarchicalNodeParser
 from llama_index.node_parser import get_leaf_nodes
 from llama_index import StorageContext
-from llama_index.retrievers import AutoMergingRetriever
-from llama_index.indices.postprocessor import SentenceTransformerRerank
-from llama_index.query_engine import RetrieverQueryEngine
-
 
+import nest_asyncio
 nest_asyncio.apply()
-openai = OpenAI()
 
+openai = OpenAI()
 qa_relevance = (
     Feedback(openai.relevance_with_cot_reasons, name="Answer Relevance")
     .on_input_output()
@@ -69,49 +60,6 @@ def get_prebuilt_trulens_recorder(query_engine, app_id):
         )
     return tru_recorder
 
-def build_sentence_window_index(
-    document, llm, embed_model="local:BAAI/bge-small-en-v1.5", save_dir="sentence_index"
-):
-    # create the sentence window node parser w/ default settings
-    node_parser = SentenceWindowNodeParser.from_defaults(
-        window_size=3,
-        window_metadata_key="window",
-        original_text_metadata_key="original_text",
-    )
-    sentence_context = ServiceContext.from_defaults(
-        llm=llm,
-        embed_model=embed_model,
-        node_parser=node_parser,
-    )
-    if not os.path.exists(save_dir):
-        sentence_index = VectorStoreIndex.from_documents(
-            [document], service_context=sentence_context
-        )
-        sentence_index.storage_context.persist(persist_dir=save_dir)
-    else:
-        sentence_index = load_index_from_storage(
-            StorageContext.from_defaults(persist_dir=save_dir),
-            service_context=sentence_context,
-        )
-
-    return sentence_index
-
-def get_sentence_window_query_engine(
-    sentence_index,
-    similarity_top_k=6,
-    rerank_top_n=2,
-):
-    # define postprocessors
-    postproc = MetadataReplacementPostProcessor(target_metadata_key="window")
-    rerank = SentenceTransformerRerank(
-        top_n=rerank_top_n, model="BAAI/bge-reranker-base"
-    )
-
-    sentence_window_engine = sentence_index.as_query_engine(
-        similarity_top_k=similarity_top_k, node_postprocessors=[postproc, rerank]
-    )
-    return sentence_window_engine
-
 def build_automerging_index(
     documents,
     llm,
@@ -140,21 +88,4 @@ def build_automerging_index(
             StorageContext.from_defaults(persist_dir=save_dir),
             service_context=merging_context,
         )
-    return automerging_index
-
-def get_automerging_query_engine(
-    automerging_index,
-    similarity_top_k=12,
-    rerank_top_n=2,
-):
-    base_retriever = automerging_index.as_retriever(similarity_top_k=similarity_top_k)
-    retriever = AutoMergingRetriever(
-        base_retriever, automerging_index.storage_context, verbose=True
-    )
-    rerank = SentenceTransformerRerank(
-        top_n=rerank_top_n, model="BAAI/bge-reranker-base"
-    )
-    auto_merging_engine = RetrieverQueryEngine.from_args(
-        retriever, node_postprocessors=[rerank]
-    )
-    return auto_merging_engine
+    return automerging_index