tweak the vision_api prompt, create configuration files, minor tweak to main script

config/model_config.yml

@@ -0,0 +1,17 @@
+input_data:
+  source:
+    - './raw_documents/qna.txt'
+    - './raw_documents/HI Chapter Summary Version 1.3.pdf'
+    - './raw_documents/conversation_examples.txt'
+    - './raw_documents/HI_Knowledge_Base.pdf'
+    - './raw_documents/answers.txt'
+
+embeddings:
+  embedding_base_model: 'BAAI/bge-small-en-v1.5'
+  fine_tuned_embedding_model: 'local:models/fine-tuned-embeddings'
+
+vector_store:
+  persisted_path: './models/chroma_db'
+
+questionaire_data:
+  db_path: './database/mock_qna.sqlite'

config/model_config_advanced.yml

@@ -0,0 +1,17 @@
+input_data:
+  source:
+    - './raw_documents/qna.txt'
+    - './raw_documents/HI Chapter Summary Version 1.3.pdf'
+    - './raw_documents/conversation_examples.txt'
+    - './raw_documents/HI_Knowledge_Base.pdf'
+    - './raw_documents/answers.txt'
+
+embeddings:
+  embedding_base_model: 'BAAI/bge-small-en-v1.5'
+  fine_tuned_embedding_model: 'local:models/fine-tuned-embeddings-advanced'
+
+vector_store:
+  persisted_path: './models/chroma_db_advanced'
+
+questionaire_data:
+  db_path: './database/mock_qna_advanced.sqlite'

models/chroma_db_advanced/a88943fe-4428-425d-8b9c-7bb8665a0c79/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2eec38a208011f4f233e59d2618152fa02e42d91757412778a5db814fe80bf2f
+size 1676000

raw_documents/overview_background.txt → models/chroma_db_advanced/a88943fe-4428-425d-8b9c-7bb8665a0c79/header.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4a5e6e0a28727dd6eab4bc18bf5ffcf897a4dbed61a854fa52629d2698f0925
-size 5970
+oid sha256:e87a1dc8bcae6f2c4bea6d5dd5005454d4dace8637dae29bff3c037ea771411e
+size 100

models/chroma_db_advanced/a88943fe-4428-425d-8b9c-7bb8665a0c79/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc19b1997119425765295aeab72d76faa6927d4f83985d328c26f20468d6cc76
+size 4000

models/chroma_db_advanced/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51aba6bb0bf5e5851de1e4e6cf53215b874c11b7194b3b765a2edfbc59ce9313
+size 15937536

models/fine-tuned-embeddings-advanced/1_Pooling/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfd7e0a022036d0ffa0f998824a918247d5a7473d968cdc92e318fd04098e682
+size 270

models/fine-tuned-embeddings-advanced/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af2a3dc885fad9e063851f6d7d61c8451bd064d9be25a3086a6f4be73e3d66ec
+size 2544

models/fine-tuned-embeddings-advanced/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d1406e6b622e1d931c5535df1578231e0b315bf77ac55d547f36faed55b99ef
+size 706

models/fine-tuned-embeddings-advanced/config_sentence_transformers.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:940d5f50db195fa6e5e6a4f122c095f77880de259d74b14a65779ed48bdd7c56
+size 124

models/fine-tuned-embeddings-advanced/eval/Information-Retrieval_evaluation_results.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6120b99457f04ca31972429df8bcdc01ea1f1789df3f3a7b90859440d23cdedf
+size 4140

models/fine-tuned-embeddings-advanced/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eed74129b591608f8b74c53a800ae0035e63d623618cb64e26638124beb54f6
+size 133462128

models/fine-tuned-embeddings-advanced/modules.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf
+size 349

models/fine-tuned-embeddings-advanced/sentence_bert_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e39fda68ccbff05bfa723ae9c0e70e23e2ec373b76e0f8c6e71af72a693cbf
+size 52

models/fine-tuned-embeddings-advanced/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a
+size 695

models/fine-tuned-embeddings-advanced/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91f1def9b9391fdabe028cd3f3fcc4efd34e5d1f08c3bf2de513ebb5911a1854
+size 711649

models/fine-tuned-embeddings-advanced/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b29c7bfc889e53b36d9dd3e686dd4300f6525110eaa98c76a5dafceb2029f53
+size 1242

models/fine-tuned-embeddings-advanced/vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3
+size 231508

notebooks/001_fine-tuning-embedding-model-advanced.ipynb

@@ -0,0 +1,1470 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ca2c990f-5215-4ab9-8143-1d79db28edc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json, os\n",
+    "\n",
+    "from llama_index.core import SimpleDirectoryReader\n",
+    "from llama_index.core.node_parser import SentenceSplitter\n",
+    "from llama_index.core.schema import MetadataMode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "139da55d-f0c3-4b76-b47f-e18ee552eb30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.finetuning.embeddings.common import (\n",
+    "    EmbeddingQAFinetuneDataset,\n",
+    "    generate_qa_embedding_pairs,\n",
+    ")\n",
+    "from llama_index.finetuning import SentenceTransformersFinetuneEngine"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1dfb1acc-606b-4106-baf7-87ed487b5d9c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.embeddings.openai.base import OpenAIEmbedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "fa06c66a-ab07-46a6-bc53-f6157017883c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core import ServiceContext, VectorStoreIndex"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "c9928491-520a-441a-8c44-1fc21cfa5def",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.schema import TextNode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "25f0c7a3-c52f-4417-aec8-4b6cfbf7a1b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm.notebook import tqdm\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "62f4d7f0-748a-405e-b5f1-6520fd02bedc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sentence_transformers.evaluation import InformationRetrievalEvaluator\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "12527049-a5cb-423c-8de5-099aee970c85",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms.openai import OpenAI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7dc65d7b-3cdb-4513-b09f-f7406ad59b35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "978cf71f-1ce7-4598-92fe-18fe22ca37c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TRAIN_FILES = [\"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "               \"../raw_documents/HI Chapter Summary Version 1.3.pdf\"]\n",
+    "VAL_FILES = [\"../raw_documents/qna.txt\",\n",
+    "             \"../raw_documents/conversation_examples.txt\",\n",
+    "             \"../raw_documents/answers.txt\"]\n",
+    "\n",
+    "### based on all docs\n",
+    "TRAIN_CORPUS_FPATH = \"../data/train_corpus_advanced.json\"\n",
+    "\n",
+    "### based on ../raw_documents/HI Chapter Summary Version 1.3.pdf\n",
+    "VAL_CORPUS_FPATH = \"../data/val_corpus.json\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "663cd20e-c16e-4dda-924e-5f60eb25a772",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "26f614c8-eb45-4cc1-b067-2c7299587982",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_corpus(files, verbose=False):\n",
+    "    if verbose:\n",
+    "        print(f\"Loading files {files}\")\n",
+    "\n",
+    "    reader = SimpleDirectoryReader(input_files=files)\n",
+    "    docs = reader.load_data()\n",
+    "    if verbose:\n",
+    "        print(f\"Loaded {len(docs)} docs\")\n",
+    "\n",
+    "    parser = SentenceSplitter()\n",
+    "    nodes = parser.get_nodes_from_documents(docs, show_progress=verbose)\n",
+    "\n",
+    "    if verbose:\n",
+    "        print(f\"Parsed {len(nodes)} nodes\")\n",
+    "\n",
+    "    return nodes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6ba52e5-4d7f-4c30-8979-8d84a1bc3ca4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "84cc4308-8ac4-4eba-9478-b81d5b645c48",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "load qa embedding training pairs from saved corpus file..\n",
+      "load qa embedding validation pairs from saved corpus file..\n"
+     ]
+    }
+   ],
+   "source": [
+    "if not os.path.exists(TRAIN_CORPUS_FPATH):\n",
+    "    train_nodes = load_corpus(TRAIN_FILES, verbose=True)\n",
+    "    print(\"generating qa embedding pairs for training data..\")\n",
+    "    train_dataset = generate_qa_embedding_pairs(\n",
+    "        llm=OpenAI(model=\"gpt-3.5-turbo-1106\"), nodes=train_nodes\n",
+    "    )\n",
+    "    train_dataset.save_json(TRAIN_CORPUS_FPATH)\n",
+    "else:\n",
+    "    print(\"load qa embedding training pairs from saved corpus file..\")\n",
+    "    train_dataset = EmbeddingQAFinetuneDataset.from_json(TRAIN_CORPUS_FPATH)\n",
+    "\n",
+    "if not os.path.exists(VAL_CORPUS_FPATH):\n",
+    "    val_nodes = load_corpus(VAL_FILES, verbose=True)\n",
+    "    print(\"generating qa embedding pairs for validation data..\")\n",
+    "    val_dataset = generate_qa_embedding_pairs(\n",
+    "        llm=OpenAI(model=\"gpt-3.5-turbo-1106\"), nodes=val_nodes\n",
+    "    )\n",
+    "    val_dataset.save_json(VAL_CORPUS_FPATH)\n",
+    "else:\n",
+    "    print(\"load qa embedding validation pairs from saved corpus file..\")\n",
+    "    val_dataset = EmbeddingQAFinetuneDataset.from_json(VAL_CORPUS_FPATH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c3399443-5936-4dfe-b0ec-821d222e734d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "8f17c832-e9ae-477b-8bf7-a9c8410f1ed8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "19241142d8534d139252ffe078559bb7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/94.8k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "finetune_engine = SentenceTransformersFinetuneEngine(\n",
+    "    train_dataset,\n",
+    "    model_id=\"BAAI/bge-small-en-v1.5\",\n",
+    "    model_output_path=\"../models/fine-tuned-embeddings-advanced\",\n",
+    "    batch_size=5,\n",
+    "    val_dataset=val_dataset\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "a6498d0b-da9a-4f7f-8c85-c9bf4d772c72",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2c10018eda384f49a220c4fa66738fe1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5f4e5628b306450eab01e3af1ebdaf28",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Iteration:   0%|          | 0/268 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bce2bb08b15548f8afd8fd878f2009a4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Iteration:   0%|          | 0/268 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "finetune_engine.finetune()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "e057b405-aa0e-4e78-91e0-9bf40f01c1a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embed_model = finetune_engine.get_finetuned_model()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "72d9f97a-0902-4e65-8459-b34613e419f6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "HuggingFaceEmbedding(model_name='../models/fine-tuned-embeddings-advanced', embed_batch_size=10, callback_manager=<llama_index.core.callbacks.base.CallbackManager object at 0x29f61adf0>, tokenizer_name='../models/fine-tuned-embeddings-advanced', max_length=512, pooling=<Pooling.CLS: 'cls'>, normalize=True, query_instruction=None, text_instruction=None, cache_folder=None)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c4f4058c-edbb-43c4-bebe-8c36d410e819",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "97ebae28-80ef-4f35-92ce-a370776e3b22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fine_tuned_embed_model = SentenceTransformer(\"../models/fine-tuned-embeddings-advanced\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dad7589f-4855-4432-b710-01aff9c134ee",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "ac4a1a5b-974d-452e-8507-0950c962f9b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def evaluate(\n",
+    "    dataset,\n",
+    "    embed_model,\n",
+    "    top_k=5,\n",
+    "    verbose=False,\n",
+    "):\n",
+    "    corpus = dataset.corpus\n",
+    "    queries = dataset.queries\n",
+    "    relevant_docs = dataset.relevant_docs\n",
+    "\n",
+    "    service_context = ServiceContext.from_defaults(embed_model=embed_model)\n",
+    "    nodes = [TextNode(id_=id_, text=text) for id_, text in corpus.items()]\n",
+    "    index = VectorStoreIndex(\n",
+    "        nodes, service_context=service_context, show_progress=True\n",
+    "    )\n",
+    "    retriever = index.as_retriever(similarity_top_k=top_k)\n",
+    "\n",
+    "    eval_results = []\n",
+    "    for query_id, query in tqdm(queries.items()):\n",
+    "        retrieved_nodes = retriever.retrieve(query)\n",
+    "        retrieved_ids = [node.node.node_id for node in retrieved_nodes]\n",
+    "        expected_id = relevant_docs[query_id][0]\n",
+    "        is_hit = expected_id in retrieved_ids  # assume 1 relevant doc\n",
+    "\n",
+    "        eval_result = {\n",
+    "            \"is_hit\": is_hit,\n",
+    "            \"retrieved\": retrieved_ids,\n",
+    "            \"expected\": expected_id,\n",
+    "            \"query\": query_id,\n",
+    "        }\n",
+    "        eval_results.append(eval_result)\n",
+    "    return eval_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "a53cf893-ce9f-4d9d-ad4a-e9e17fb058d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def evaluate_st(\n",
+    "    dataset,\n",
+    "    model_id,\n",
+    "    name,\n",
+    "):\n",
+    "    corpus = dataset.corpus\n",
+    "    queries = dataset.queries\n",
+    "    relevant_docs = dataset.relevant_docs\n",
+    "\n",
+    "    evaluator = InformationRetrievalEvaluator(\n",
+    "        queries, corpus, relevant_docs, name=name\n",
+    "    )\n",
+    "    model = SentenceTransformer(model_id)\n",
+    "    output_path = \"../results/\"\n",
+    "    Path(output_path).mkdir(exist_ok=True, parents=True)\n",
+    "    return evaluator(model, output_path=output_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "703f9350-f7ab-43cc-abdf-055323ef67dd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "57d66621-49e6-4a8a-9ef2-83b2b33e33d7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b43ad08e-e96d-412b-9a88-14fe3af85b3d",
+   "metadata": {},
+   "source": [
+    "### Using OpenAI Ada embedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "91f057aa-4b59-48ea-b3d5-23012a4d487f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/9p/zqv8rk793ts9cxxfr66p40sh0000gn/T/ipykernel_34681/2760886022.py:11: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
+      "  service_context = ServiceContext.from_defaults(embed_model=embed_model)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3cd092342b1846ed9aa81f8de44eaaea",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating embeddings:   0%|          | 0/100 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "00a72686c4bc4e518e8c7f56124247ab",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/200 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "ada = OpenAIEmbedding()\n",
+    "ada_val_results = evaluate(val_dataset, ada)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "5d2f59c6-75d3-4970-bac3-dfe0eef00efe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_ada = pd.DataFrame(ada_val_results)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "7a697cd8-6f39-4d5b-84f4-f08cf58adc4a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>is_hit</th>\n",
+       "      <th>retrieved</th>\n",
+       "      <th>expected</th>\n",
+       "      <th>query</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>False</td>\n",
+       "      <td>[5b9cd986-33dc-46f1-abae-e4e1dc9e3629, c3c1804...</td>\n",
+       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
+       "      <td>011d84b2-0c26-4c5c-89d1-2a85498f30e0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[6a756f03-638d-480d-8222-1a6bf3790e3c, c3c1804...</td>\n",
+       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
+       "      <td>70c5ddd7-eb86-4a41-af70-a23d2392f48d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...</td>\n",
+       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
+       "      <td>a8f4290a-1281-4272-aab9-bf089954a45e</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...</td>\n",
+       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
+       "      <td>c1ef991a-1cc6-4dbf-b179-2df688c84301</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...</td>\n",
+       "      <td>21778248-2ed9-4147-bdb0-a60337a1a599</td>\n",
+       "      <td>1ce25e78-c1e1-487e-9455-9418baa0b60c</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   is_hit                                          retrieved  \\\n",
+       "0   False  [5b9cd986-33dc-46f1-abae-e4e1dc9e3629, c3c1804...   \n",
+       "1    True  [6a756f03-638d-480d-8222-1a6bf3790e3c, c3c1804...   \n",
+       "2    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...   \n",
+       "3    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...   \n",
+       "4    True  [21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...   \n",
+       "\n",
+       "                               expected                                 query  \n",
+       "0  6a756f03-638d-480d-8222-1a6bf3790e3c  011d84b2-0c26-4c5c-89d1-2a85498f30e0  \n",
+       "1  6a756f03-638d-480d-8222-1a6bf3790e3c  70c5ddd7-eb86-4a41-af70-a23d2392f48d  \n",
+       "2  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  a8f4290a-1281-4272-aab9-bf089954a45e  \n",
+       "3  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  c1ef991a-1cc6-4dbf-b179-2df688c84301  \n",
+       "4  21778248-2ed9-4147-bdb0-a60337a1a599  1ce25e78-c1e1-487e-9455-9418baa0b60c  "
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_ada[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "3f7186fb-f392-4531-8959-25161e3905e4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(0.95, 200)"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "hit_rate_ada = df_ada[\"is_hit\"].mean()\n",
+    "hit_rate_ada, len(df_ada)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d044399a-e55b-40b7-a09d-6fb838383bfa",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "66746f3e-638a-432c-a38d-7cb99d2093f7",
+   "metadata": {},
+   "source": [
+    "### Using BAAI bge-small model without fine-tuning"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "b2905831-0eb9-4ea7-a0b9-5db286b0965e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/9p/zqv8rk793ts9cxxfr66p40sh0000gn/T/ipykernel_34681/2760886022.py:11: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
+      "  service_context = ServiceContext.from_defaults(embed_model=embed_model)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ca1ac4b4b54f4169b909e5633b3eb1ad",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating embeddings:   0%|          | 0/100 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4293592aba3244a991fad843f5c881ba",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/200 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "bge = \"local:BAAI/bge-small-en-v1.5\"\n",
+    "bge_val_results = evaluate(val_dataset, bge)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "4e66270d-d3f6-429e-9e48-e8062866aa02",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_bge = pd.DataFrame(bge_val_results)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "698c1eb7-eba4-4383-98aa-931fc4ad56a4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>is_hit</th>\n",
+       "      <th>retrieved</th>\n",
+       "      <th>expected</th>\n",
+       "      <th>query</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>False</td>\n",
+       "      <td>[69a5696d-0c0e-482a-b6a9-f7b87f19945f, fa650c7...</td>\n",
+       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
+       "      <td>011d84b2-0c26-4c5c-89d1-2a85498f30e0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[6a756f03-638d-480d-8222-1a6bf3790e3c, d89a649...</td>\n",
+       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
+       "      <td>70c5ddd7-eb86-4a41-af70-a23d2392f48d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...</td>\n",
+       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
+       "      <td>a8f4290a-1281-4272-aab9-bf089954a45e</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, ad2e3eb...</td>\n",
+       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
+       "      <td>c1ef991a-1cc6-4dbf-b179-2df688c84301</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>True</td>\n",
+       "      <td>[21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...</td>\n",
+       "      <td>21778248-2ed9-4147-bdb0-a60337a1a599</td>\n",
+       "      <td>1ce25e78-c1e1-487e-9455-9418baa0b60c</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   is_hit                                          retrieved  \\\n",
+       "0   False  [69a5696d-0c0e-482a-b6a9-f7b87f19945f, fa650c7...   \n",
+       "1    True  [6a756f03-638d-480d-8222-1a6bf3790e3c, d89a649...   \n",
+       "2    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...   \n",
+       "3    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, ad2e3eb...   \n",
+       "4    True  [21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...   \n",
+       "\n",
+       "                               expected                                 query  \n",
+       "0  6a756f03-638d-480d-8222-1a6bf3790e3c  011d84b2-0c26-4c5c-89d1-2a85498f30e0  \n",
+       "1  6a756f03-638d-480d-8222-1a6bf3790e3c  70c5ddd7-eb86-4a41-af70-a23d2392f48d  \n",
+       "2  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  a8f4290a-1281-4272-aab9-bf089954a45e  \n",
+       "3  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  c1ef991a-1cc6-4dbf-b179-2df688c84301  \n",
+       "4  21778248-2ed9-4147-bdb0-a60337a1a599  1ce25e78-c1e1-487e-9455-9418baa0b60c  "
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_bge[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "9b1cb546-4605-4c48-bf4e-df812db97f13",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(0.915, 200)"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "hit_rate_bge = df_bge[\"is_hit\"].mean()\n",
+    "hit_rate_bge, len(df_bge)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7dd69ad1-2153-4df0-93f7-807fc289d3fd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "1b12ca3d-6ca2-41f6-9ddb-b12b9354ca83",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.7955697668171072"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "evaluate_st(val_dataset, \"BAAI/bge-small-en-v1.5\", name=\"bge\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6023382b-0ff5-4d60-aeac-ad523153f943",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adf35a2a-3bb7-4251-9521-f35346a7c6e6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b3d290c2-784f-4c41-a258-e11d2c5117e7",
+   "metadata": {},
+   "source": [
+    "### Using BAAI bge-small model with `fine-tuning`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "bd42b288-1f1f-41aa-9fd4-1ae4b1df462b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/9p/zqv8rk793ts9cxxfr66p40sh0000gn/T/ipykernel_34681/2760886022.py:11: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
+      "  service_context = ServiceContext.from_defaults(embed_model=embed_model)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9ddb31814f674c658e4b509c45104c7a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating embeddings:   0%|          | 0/100 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6e781eff650b4cd28345ed4a0c919a28",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/200 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "finetuned = \"local:../models/fine-tuned-embeddings-advanced\"\n",
+    "val_results_finetuned = evaluate(val_dataset, finetuned)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "b1d7112d-b1b8-47db-8a4b-6c024ef99dd6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_finetuned = pd.DataFrame(val_results_finetuned)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "62a4dd29-0631-4c5b-88e1-be43d48e1043",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.97"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "hit_rate_finetuned = df_finetuned[\"is_hit\"].mean()\n",
+    "hit_rate_finetuned"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "4332594b-c861-40fb-a58b-ba36717d0519",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.8835191391941393"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "evaluate_st(val_dataset, \"../models/fine-tuned-embeddings-advanced\", name=\"finetuned\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b0003812-84a2-4ebd-9372-07bf874a486b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ae7eb6ff-181b-42c8-975c-ca3320158698",
+   "metadata": {},
+   "source": [
+    "### Summary"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "3ca46cff-b186-463a-847d-a86c310268ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_ada[\"model\"] = \"ada\"\n",
+    "df_bge[\"model\"] = \"bge\"\n",
+    "df_finetuned[\"model\"] = \"fine_tuned\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "d1d3053e-2395-48a0-af59-fd27180e1e7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>is_hit</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>model</th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>ada</th>\n",
+       "      <td>0.950</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>bge</th>\n",
+       "      <td>0.915</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fine_tuned</th>\n",
+       "      <td>0.970</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            is_hit\n",
+       "model             \n",
+       "ada          0.950\n",
+       "bge          0.915\n",
+       "fine_tuned   0.970"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_all = pd.concat([df_ada, df_bge, df_finetuned])\n",
+    "df_all.groupby(\"model\").mean(\"is_hit\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72575c28-a221-4967-8f04-9579dcefa8f8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "id": "032cac38-c856-4aeb-9bbb-6d70ed53c614",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_st_bge = pd.read_csv(\n",
+    "    \"../results/Information-Retrieval_evaluation_bge_results.csv\"\n",
+    ")\n",
+    "df_st_finetuned = pd.read_csv(\n",
+    "    \"../results/Information-Retrieval_evaluation_finetuned_results.csv\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a509f239-8b28-4d0a-9101-c8de91c7943b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "d2975262-c486-4a9a-a61f-ea535203a0f3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>epoch</th>\n",
+       "      <th>steps</th>\n",
+       "      <th>cos_sim-Accuracy@1</th>\n",
+       "      <th>cos_sim-Accuracy@3</th>\n",
+       "      <th>cos_sim-Accuracy@5</th>\n",
+       "      <th>cos_sim-Accuracy@10</th>\n",
+       "      <th>cos_sim-Precision@1</th>\n",
+       "      <th>cos_sim-Recall@1</th>\n",
+       "      <th>cos_sim-Precision@3</th>\n",
+       "      <th>cos_sim-Recall@3</th>\n",
+       "      <th>...</th>\n",
+       "      <th>dot_score-Recall@1</th>\n",
+       "      <th>dot_score-Precision@3</th>\n",
+       "      <th>dot_score-Recall@3</th>\n",
+       "      <th>dot_score-Precision@5</th>\n",
+       "      <th>dot_score-Recall@5</th>\n",
+       "      <th>dot_score-Precision@10</th>\n",
+       "      <th>dot_score-Recall@10</th>\n",
+       "      <th>dot_score-MRR@10</th>\n",
+       "      <th>dot_score-NDCG@10</th>\n",
+       "      <th>dot_score-MAP@100</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>model</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>bge</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>0.920</td>\n",
+       "      <td>0.96</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.288333</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.288333</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>0.184</td>\n",
+       "      <td>0.920</td>\n",
+       "      <td>0.096</td>\n",
+       "      <td>0.96</td>\n",
+       "      <td>0.792935</td>\n",
+       "      <td>0.833595</td>\n",
+       "      <td>0.795570</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>bge</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>0.920</td>\n",
+       "      <td>0.96</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.288333</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.288333</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>0.184</td>\n",
+       "      <td>0.920</td>\n",
+       "      <td>0.096</td>\n",
+       "      <td>0.96</td>\n",
+       "      <td>0.792935</td>\n",
+       "      <td>0.833595</td>\n",
+       "      <td>0.795570</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>bge</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>0.920</td>\n",
+       "      <td>0.96</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.288333</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.288333</td>\n",
+       "      <td>0.865</td>\n",
+       "      <td>0.184</td>\n",
+       "      <td>0.920</td>\n",
+       "      <td>0.096</td>\n",
+       "      <td>0.96</td>\n",
+       "      <td>0.792935</td>\n",
+       "      <td>0.833595</td>\n",
+       "      <td>0.795570</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fine_tuned</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.900</td>\n",
+       "      <td>0.970</td>\n",
+       "      <td>0.98</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.300000</td>\n",
+       "      <td>0.900</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.300000</td>\n",
+       "      <td>0.900</td>\n",
+       "      <td>0.194</td>\n",
+       "      <td>0.970</td>\n",
+       "      <td>0.098</td>\n",
+       "      <td>0.98</td>\n",
+       "      <td>0.856264</td>\n",
+       "      <td>0.886738</td>\n",
+       "      <td>0.857339</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fine_tuned</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.900</td>\n",
+       "      <td>0.970</td>\n",
+       "      <td>0.98</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.300000</td>\n",
+       "      <td>0.900</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.790</td>\n",
+       "      <td>0.300000</td>\n",
+       "      <td>0.900</td>\n",
+       "      <td>0.194</td>\n",
+       "      <td>0.970</td>\n",
+       "      <td>0.098</td>\n",
+       "      <td>0.98</td>\n",
+       "      <td>0.856264</td>\n",
+       "      <td>0.886738</td>\n",
+       "      <td>0.857339</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fine_tuned</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.770</td>\n",
+       "      <td>0.910</td>\n",
+       "      <td>0.965</td>\n",
+       "      <td>0.98</td>\n",
+       "      <td>0.770</td>\n",
+       "      <td>0.770</td>\n",
+       "      <td>0.303333</td>\n",
+       "      <td>0.910</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.770</td>\n",
+       "      <td>0.303333</td>\n",
+       "      <td>0.910</td>\n",
+       "      <td>0.193</td>\n",
+       "      <td>0.965</td>\n",
+       "      <td>0.098</td>\n",
+       "      <td>0.98</td>\n",
+       "      <td>0.847542</td>\n",
+       "      <td>0.880388</td>\n",
+       "      <td>0.848711</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fine_tuned</th>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>0.815</td>\n",
+       "      <td>0.945</td>\n",
+       "      <td>0.970</td>\n",
+       "      <td>0.99</td>\n",
+       "      <td>0.815</td>\n",
+       "      <td>0.815</td>\n",
+       "      <td>0.315000</td>\n",
+       "      <td>0.945</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.815</td>\n",
+       "      <td>0.315000</td>\n",
+       "      <td>0.945</td>\n",
+       "      <td>0.194</td>\n",
+       "      <td>0.970</td>\n",
+       "      <td>0.099</td>\n",
+       "      <td>0.99</td>\n",
+       "      <td>0.882935</td>\n",
+       "      <td>0.909563</td>\n",
+       "      <td>0.883519</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>7 rows × 32 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            epoch  steps  cos_sim-Accuracy@1  cos_sim-Accuracy@3  \\\n",
+       "model                                                              \n",
+       "bge            -1     -1               0.705               0.865   \n",
+       "bge            -1     -1               0.705               0.865   \n",
+       "bge            -1     -1               0.705               0.865   \n",
+       "fine_tuned     -1     -1               0.790               0.900   \n",
+       "fine_tuned     -1     -1               0.790               0.900   \n",
+       "fine_tuned     -1     -1               0.770               0.910   \n",
+       "fine_tuned     -1     -1               0.815               0.945   \n",
+       "\n",
+       "            cos_sim-Accuracy@5  cos_sim-Accuracy@10  cos_sim-Precision@1  \\\n",
+       "model                                                                      \n",
+       "bge                      0.920                 0.96                0.705   \n",
+       "bge                      0.920                 0.96                0.705   \n",
+       "bge                      0.920                 0.96                0.705   \n",
+       "fine_tuned               0.970                 0.98                0.790   \n",
+       "fine_tuned               0.970                 0.98                0.790   \n",
+       "fine_tuned               0.965                 0.98                0.770   \n",
+       "fine_tuned               0.970                 0.99                0.815   \n",
+       "\n",
+       "            cos_sim-Recall@1  cos_sim-Precision@3  cos_sim-Recall@3  ...  \\\n",
+       "model                                                                ...   \n",
+       "bge                    0.705             0.288333             0.865  ...   \n",
+       "bge                    0.705             0.288333             0.865  ...   \n",
+       "bge                    0.705             0.288333             0.865  ...   \n",
+       "fine_tuned             0.790             0.300000             0.900  ...   \n",
+       "fine_tuned             0.790             0.300000             0.900  ...   \n",
+       "fine_tuned             0.770             0.303333             0.910  ...   \n",
+       "fine_tuned             0.815             0.315000             0.945  ...   \n",
+       "\n",
+       "            dot_score-Recall@1  dot_score-Precision@3  dot_score-Recall@3  \\\n",
+       "model                                                                       \n",
+       "bge                      0.705               0.288333               0.865   \n",
+       "bge                      0.705               0.288333               0.865   \n",
+       "bge                      0.705               0.288333               0.865   \n",
+       "fine_tuned               0.790               0.300000               0.900   \n",
+       "fine_tuned               0.790               0.300000               0.900   \n",
+       "fine_tuned               0.770               0.303333               0.910   \n",
+       "fine_tuned               0.815               0.315000               0.945   \n",
+       "\n",
+       "            dot_score-Precision@5  dot_score-Recall@5  dot_score-Precision@10  \\\n",
+       "model                                                                           \n",
+       "bge                         0.184               0.920                   0.096   \n",
+       "bge                         0.184               0.920                   0.096   \n",
+       "bge                         0.184               0.920                   0.096   \n",
+       "fine_tuned                  0.194               0.970                   0.098   \n",
+       "fine_tuned                  0.194               0.970                   0.098   \n",
+       "fine_tuned                  0.193               0.965                   0.098   \n",
+       "fine_tuned                  0.194               0.970                   0.099   \n",
+       "\n",
+       "            dot_score-Recall@10  dot_score-MRR@10  dot_score-NDCG@10  \\\n",
+       "model                                                                  \n",
+       "bge                        0.96          0.792935           0.833595   \n",
+       "bge                        0.96          0.792935           0.833595   \n",
+       "bge                        0.96          0.792935           0.833595   \n",
+       "fine_tuned                 0.98          0.856264           0.886738   \n",
+       "fine_tuned                 0.98          0.856264           0.886738   \n",
+       "fine_tuned                 0.98          0.847542           0.880388   \n",
+       "fine_tuned                 0.99          0.882935           0.909563   \n",
+       "\n",
+       "            dot_score-MAP@100  \n",
+       "model                          \n",
+       "bge                  0.795570  \n",
+       "bge                  0.795570  \n",
+       "bge                  0.795570  \n",
+       "fine_tuned           0.857339  \n",
+       "fine_tuned           0.857339  \n",
+       "fine_tuned           0.848711  \n",
+       "fine_tuned           0.883519  \n",
+       "\n",
+       "[7 rows x 32 columns]"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_st_bge[\"model\"] = \"bge\"\n",
+    "df_st_finetuned[\"model\"] = \"fine_tuned\"\n",
+    "df_st_all = pd.concat([df_st_bge, df_st_finetuned])\n",
+    "df_st_all = df_st_all.set_index(\"model\")\n",
+    "df_st_all"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6ed2321b-6618-4a2b-9b1c-028425e91b84",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/002_persisted-embedding-model-advanced.ipynb

@@ -0,0 +1,507 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8acae3ed-2953-45a3-aba9-0327b6ae3679",
+   "metadata": {},
+   "source": [
+    "### ChromaDB method - create vectorstore based on Chroma"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7de9c591-5a77-4bbe-80f1-4897e15f0b97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.core import ServiceContext\n",
+    "from llama_index.core import Document\n",
+    "\n",
+    "from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e65dff6-77b6-4be8-8857-5cecf3a035bb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load some documents\n",
+    "documents = SimpleDirectoryReader(input_files=[\n",
+    "                                    \"../raw_documents/qna.txt\",\n",
+    "                                    \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "                                    \"../raw_documents/conversation_examples.txt\",\n",
+    "                                    \"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "                                    \"../raw_documents/answers.txt\",\n",
+    "                                  ]).load_data()\n",
+    "document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bd86b3f5-1dfc-4257-bd9c-86d34f02398d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# initialize client, setting path to save data\n",
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db_advanced\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f568ce7b-bcbf-455c-acf1-6c2cae129fed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create collection\n",
+    "chroma_collection = db.get_or_create_collection(\"quickstart\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ed0b018e-1982-46b2-b1b4-04f5c0ce8672",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# assign chroma as the vector_store to the context\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eb5edab2-30db-4bf7-96b5-4005d3161988",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0946b6ce-96ab-44de-ad75-e424a8429f67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Settings.llm = None\n",
+    "Settings.chunk_size = 1024\n",
+    "Settings.embed_model = \"local:../models/fine-tuned-embeddings-advanced\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b8c73a2c-1129-406a-8046-085afcaf9cbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nodes = Settings.node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75f1c76f-d3e5-4b69-818c-98865adb1457",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adfe688f-95c0-477c-a9de-e9e77541a1d7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dab4c6f3-ef67-4d90-b3d5-e290c5d1b6f4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6a764113-ad7e-4674-aa57-ebbf405902a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context.docstore.add_documents(nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38e7c88d-6c45-4275-8293-d09b4b85a7cf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e492ed4a-23a3-47d6-8b50-51fb48b3aa05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start_time = time.time()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cbd11b89-9b83-4f08-bb30-160f750f2ffb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_index = VectorStoreIndex(nodes, storage_context=storage_context)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "082a0d7e-b025-4db1-be2a-7a0b7bc453b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_query_engine = vector_index.as_query_engine()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d3bd848d-9985-4a3d-bdc4-ec340cc69ef3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "indexing_cost = time.time() - start_time\n",
+    "indexing_cost = indexing_cost / 60\n",
+    "print(f\"Indexing time: {indexing_cost:.1f} mins\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3290e870-41d7-49c4-9c4f-cb16bd1f469e",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "response = vector_query_engine.query(\"Healthcare System in Singapore consists of?\")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "131d907a-0677-4ad8-b3f7-6fc9b9c5d0a5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "08fb2be5-3a44-4bb8-a9fc-61d7f03b7a35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7fc01f6-4738-415b-a96b-afd6cf8d789a",
+   "metadata": {},
+   "source": [
+    "### ChromaDB method - load vectorstore based on Chroma"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1a42c35-5f57-423c-8fb7-7d18b3b466b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.core import ServiceContext\n",
+    "from llama_index.core import Document\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.core.memory import ChatMemoryBuffer\n",
+    "\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72dd0ece-c72d-428a-89b4-9494d948c845",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d38dc953-b923-4128-86a1-c8c6f69af0ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fine_tuned_path = \"local:../models/fine-tuned-embeddings-advanced\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4c83c613-2cfc-4871-9d07-c82f77a3bd5e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(model=\"gpt-4-0125-preview\", temperature=0.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0583e9b0-d977-488c-8331-46dfa749924c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Settings.llm = llm\n",
+    "Settings.embed_model = fine_tuned_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f994f440-f647-48b4-a517-46a79f7561e5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2159a2b6-494b-41b9-ac54-dd342bfb74ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db_advanced\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b385644-b46e-4d13-88fa-9f4af39db405",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chroma_collection = db.get_or_create_collection(\"quickstart\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93cb53d1-6b8c-4b2d-a839-53501c0d54b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# assign chroma as the vector_store to the context\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c40d59e1-6d42-41f0-8c9b-70aa026093ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create your index\n",
+    "index = VectorStoreIndex.from_vector_store(\n",
+    "    vector_store=vector_store,\n",
+    "    storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73ba6d06-ba69-4b5e-962a-9cf7d2dc4d94",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a506940-c2b4-4d14-ad93-fd451331c582",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_content = (\"You are a helpful study assistant. \"\n",
+    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
+    "                  \"You only respond once as 'Assistant'.\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3f592848-8536-4b4d-b34a-adc32d043432",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "memory = ChatMemoryBuffer.from_defaults(token_limit=100_000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6c7df81a-fd2f-42bf-b09c-46d7750f7252",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chat_engine = index.as_chat_engine(\n",
+    "    chat_mode=\"context\",\n",
+    "    memory=memory,\n",
+    "    system_prompt=system_content\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "434f0caf-8b1f-40c6-b9ec-b039cd1ca612",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"\"\"\n",
+    "Question: Which of the following is NOT a characteristic of medical expense insurance?\n",
+    "A. Pro ration factor and co-insurance.\n",
+    "B. Deductibles apply for all treatments.\n",
+    "C. Impose Sub- Limits.\n",
+    "D. Can be issued as a rider or stand-alone.\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "78abaf95-e52d-445c-9d8e-bc51efb20f06",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = chat_engine.chat(prompt)\n",
+    "print(res.response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e62303c-3a00-448f-ad93-15cb6cee1f24",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dad72f9f-7f86-407d-93be-f5724cb30d5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_engine = index.as_query_engine(\n",
+    "                memory=memory,\n",
+    "                system_prompt=system_content,\n",
+    "                similarity_top_k=3,\n",
+    "                streaming=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab778a5d-d438-4f39-88f5-c67a1f1d575e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7bb7c21a-7461-40c1-87a7-4a1f92f70153",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = hi_engine.query(\"may I know what is the rationale?\")\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "874a39ce-e682-42fa-8085-646bacea6cdb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "301e8270-783d-4942-a05f-9683ca96fbda",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/002_persisted-embedding-model.ipynb

@@ -271,7 +271,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "llm = OpenAI(model=\"gpt-3.5-turbo-1106\", temperature=0.0)"
+    "llm = OpenAI(model=\"gpt-4-0125-preview\", temperature=0.0)"
    ]
   },
   {
@@ -391,7 +391,23 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = chat_engine.chat(\"what is the healthcare philosophy in singapore\")\n",
+    "prompt = \"\"\"\n",
+    "Question: Which of the following is NOT a characteristic of medical expense insurance?\n",
+    "A. Pro ration factor and co-insurance.\n",
+    "B. Deductibles apply for all treatments.\n",
+    "C. Impose Sub- Limits.\n",
+    "D. Can be issued as a rider or stand-alone.\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9563515b-8a95-4dc8-a312-f57f9b59da86",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = chat_engine.chat(prompt)\n",
     "print(res.response)"
    ]
   },
@@ -413,7 +429,7 @@
     "hi_engine = index.as_query_engine(\n",
     "                memory=memory,\n",
     "                system_prompt=system_content,\n",
-    "                similarity_top_k=3,\n",
+    "                similarity_top_k=10,\n",
     "                streaming=True\n",
     ")"
    ]
@@ -433,7 +449,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = hi_engine.query(\"What is llama2?\")\n",
+    "res = hi_engine.query(prompt)\n",
     "print(res)"
    ]
   },

raw_documents/answers.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d01aaa6a0000c46cf93b1572ad15464480260dbc8fa8dc718f4718a3ba7598
+size 41317

raw_documents/conversation_examples.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd354c1b6691627a6598f124f76ef43d29a1c7108124d8d833180b8efbd207a4
+size 47902

raw_documents/qna.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da72ca56312ecb78d7cf6c9288b16a520baa2286136b4677cf09f36ee4f07b36
-size 56792
+oid sha256:62f7746092d2d52d8028fb13471427e220aae0ab411771eda56883e9bfdc75ce
+size 75976

requirements.txt

@@ -1,3 +1,4 @@
+aenum==3.1.15
 aiohttp==3.9.1
 aiosignal==1.3.1
 alembic==1.13.1
@@ -28,6 +29,7 @@ charset-normalizer==3.3.2
 chroma-hnswlib==0.7.3
 chromadb==0.4.22
 click==8.1.7
+cohere==4.49
 coloredlogs==15.0.1
 comm==0.2.0
 contourpy==1.2.0
@@ -45,6 +47,7 @@ exceptiongroup==1.2.0
 executing==2.0.1
 Faker==22.0.0
 fastapi==0.109.0
+fastavro==1.9.1
 fastjsonschema==2.19.1
 favicon==0.7.0
 filelock==3.13.1
@@ -58,6 +61,7 @@ gitdb==4.0.11
 GitPython==3.1.40
 google-auth==2.27.0
 googleapis-common-protos==1.62.0
+gradientai==1.7.0
 greenlet==3.0.3
 grpcio==1.60.0
 h11==0.14.0
@@ -101,19 +105,28 @@ langchain==0.0.354
 langchain-community==0.0.8
 langchain-core==0.1.23
 langsmith==0.0.87
-llama-index==0.10.1
-llama-index-agent-openai==0.1.1
-llama-index-core==0.10.1
+llama-index==0.10.12
+llama-index-agent-openai==0.1.5
+llama-index-cli==0.1.5
+llama-index-core==0.10.12
+llama-index-embeddings-adapter==0.1.3
 llama-index-embeddings-huggingface==0.1.1
-llama-index-embeddings-openai==0.1.1
+llama-index-embeddings-openai==0.1.6
+llama-index-finetuning==0.1.4
+llama-index-indices-managed-llama-cloud==0.1.3
 llama-index-legacy==0.9.48
-llama-index-llms-openai==0.1.1
-llama-index-multi-modal-llms-openai==0.1.1
+llama-index-llms-gradient==0.1.2
+llama-index-llms-openai==0.1.6
+llama-index-multi-modal-llms-openai==0.1.4
 llama-index-packs-auto-merging-retriever==0.1.2
-llama-index-program-openai==0.1.1
-llama-index-question-gen-openai==0.1.1
-llama-index-readers-file==0.1.2
+llama-index-postprocessor-cohere-rerank==0.1.2
+llama-index-program-openai==0.1.4
+llama-index-question-gen-openai==0.1.3
+llama-index-readers-file==0.1.5
+llama-index-readers-llama-parse==0.1.3
 llama-index-vector-stores-chroma==0.1.1
+llama-parse==0.3.4
+llamaindex-py-client==0.1.13
 lxml==5.1.0
 Mako==1.3.0
 Markdown==3.5.1
@@ -176,7 +189,7 @@ pyarrow==14.0.2
 pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
-pydantic==2.5.3
+pydantic==1.10.14
 pydantic_core==2.14.6
 pydeck==0.8.1b0
 Pygments==2.17.2
@@ -268,4 +281,4 @@ websockets==12.0
 widgetsnbextension==4.0.9
 wrapt==1.16.0
 yarl==1.9.4
-zipp==3.17.0
+zipp==3.17.0

streamlit_app.py

@@ -7,6 +7,7 @@ import base64
 from io import BytesIO
 import sqlite3
 import uuid
+import yaml
 
 import chromadb
 from llama_index.core import (
@@ -39,14 +40,14 @@ nest_asyncio.apply()
 st.set_page_config(page_title="🐻📚 Study Bear 🍯")
 openai_api = os.getenv("OPENAI_API_KEY")
 
-# "./raw_documents/HI_Knowledge_Base.pdf"
-image_prompt = False
-input_files = ["./raw_documents/HI Chapter Summary Version 1.3.pdf",
-               "./raw_documents/qna.txt"]
-embedding_model = "BAAI/bge-small-en-v1.5"
-persisted_vector_db = "./models/chroma_db"
-fine_tuned_path = "local:models/fine-tuned-embeddings"
-questionaire_db_path = "./database/mock_qna.sqlite"
+with open("./config/model_config.yml", "r") as file_reader:
+    model_config = yaml.safe_load(file_reader)
+
+input_files = model_config["input_data"]["source"]
+embedding_model = model_config["embeddings"]["embedding_base_model"]
+fine_tuned_path = model_config["embeddings"]["fine_tuned_embedding_model"]
+persisted_vector_db = model_config["vector_store"]["persisted_path"]
+questionaire_db_path = model_config["questionaire_data"]["db_path"]
 
 data_df = pd.DataFrame(
     {
@@ -109,6 +110,9 @@ if "init" not in st.session_state.keys():
     st.session_state.init = {"warm_started": "No"}
     st.session_state.feedback = False
 
+if "image_prompt" not in st.session_state.keys():
+    st.session_state.image_prompt = False
+
 # Store LLM generated responses
 if "messages" not in st.session_state.keys():
     st.session_state.messages = [{"role": "assistant", 
@@ -341,19 +345,19 @@ if prompt := st.chat_input(disabled=not openai_api):
 # Retrieve text prompt from image submission
 if prompt is None and \
    st.session_state.messages[-1]["role"] == "admin":
-    image_prompt = True
+    st.session_state.image_prompt = True
     prompt = st.session_state.messages[-1]["content"]
 
 # Generate a new response if last message is not from assistant
 if st.session_state.messages[-1]["role"] != "assistant":
     with st.chat_message("assistant", avatar=bear_img_path):
         with st.spinner("🧸💤 Thinking... 🐻💭"):
-            if image_prompt:
+            if st.session_state.image_prompt:
                 response = generate_llm_response(
                                 prompt, 
                                 tool_choice="health_insurance_textbook_query_engine"
                             )
-                image_prompt = False
+                st.session_state.image_prompt = False
             else:
                 response = generate_llm_response(prompt, tool_choice="auto")
             placeholder = st.empty()

vision_api.py

@@ -9,6 +9,14 @@ def get_transcribed_text(base64_image):
         "Content-Type": "application/json",
         "Authorization": f"Bearer {OPENAI_API_KEY}"
     }
+    image_prompt = (
+        "Understand and interpret the image properly, there could be "
+        "handwritten notes or scribbles beside the electronic text. "
+        "Once you have sufficient understanding of the image, "
+        "transcribed them into text. If the content is a question, "
+        "convert the question into text."
+    )
+    print(image_prompt)
 
     payload = {
         "model": "gpt-4-vision-preview",
@@ -18,7 +26,7 @@ def get_transcribed_text(base64_image):
                 "content": [
                     {
                     "type": "text",
-                    "text": "transcribe the image into text for me."
+                    "text": image_prompt
                     },
                     {
                     "type": "image_url",