{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'LightRAG' already exists and is not an empty directory.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/HKUDS/LightRAG.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/mayaraayat/Desktop/axa-fall2024-p2/LightRAG\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mayaraayat/Library/Python/3.11/lib/python/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "cd LightRAG\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obtaining file:///Users/mayaraayat/Desktop/axa-fall2024-p2/LightRAG\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: accelerate in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (1.1.1)\n",
      "Requirement already satisfied: aioboto3 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (13.2.0)\n",
      "Requirement already satisfied: aiohttp in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (3.11.2)\n",
      "Requirement already satisfied: graspologic in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (3.4.1)\n",
      "Requirement already satisfied: hnswlib in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (0.8.0)\n",
      "Requirement already satisfied: nano-vectordb in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (0.0.4.3)\n",
      "Requirement already satisfied: neo4j in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (5.26.0)\n",
      "Requirement already satisfied: networkx in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (3.4.2)\n",
      "Requirement already satisfied: ollama in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (0.4.0)\n",
      "Requirement already satisfied: openai in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (1.54.4)\n",
      "Requirement already satisfied: oracledb in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (2.5.0)\n",
      "Requirement already satisfied: pyvis in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (0.3.2)\n",
      "Requirement already satisfied: tenacity in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (9.0.0)\n",
      "Requirement already satisfied: tiktoken in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (0.8.0)\n",
      "Requirement already satisfied: torch in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (2.5.1)\n",
      "Requirement already satisfied: transformers in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (4.46.3)\n",
      "Requirement already satisfied: xxhash in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from lightrag-hku==1.0.1) (3.5.0)\n",
      "Requirement already satisfied: huggingface-hub>=0.21.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from accelerate->lightrag-hku==1.0.1) (0.26.2)\n",
      "Requirement already satisfied: numpy<3.0.0,>=1.17 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from accelerate->lightrag-hku==1.0.1) (1.26.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from accelerate->lightrag-hku==1.0.1) (24.1)\n",
      "Requirement already satisfied: psutil in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from accelerate->lightrag-hku==1.0.1) (6.1.0)\n",
      "Requirement already satisfied: pyyaml in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from accelerate->lightrag-hku==1.0.1) (6.0.2)\n",
      "Requirement already satisfied: safetensors>=0.4.3 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from accelerate->lightrag-hku==1.0.1) (0.4.5)\n",
      "Requirement already satisfied: filelock in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from torch->lightrag-hku==1.0.1) (3.16.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from torch->lightrag-hku==1.0.1) (4.12.2)\n",
      "Requirement already satisfied: jinja2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from torch->lightrag-hku==1.0.1) (3.1.4)\n",
      "Requirement already satisfied: fsspec in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from torch->lightrag-hku==1.0.1) (2024.9.0)\n",
      "Requirement already satisfied: sympy==1.13.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from torch->lightrag-hku==1.0.1) (1.13.1)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from sympy==1.13.1->torch->lightrag-hku==1.0.1) (1.3.0)\n",
      "Requirement already satisfied: aiobotocore==2.15.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (2.15.2)\n",
      "Requirement already satisfied: aiofiles>=23.2.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aioboto3->lightrag-hku==1.0.1) (23.2.1)\n",
      "Requirement already satisfied: botocore<1.35.37,>=1.35.16 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiobotocore==2.15.2->aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (1.35.36)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.10.10 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiobotocore==2.15.2->aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (1.16.0)\n",
      "Requirement already satisfied: aioitertools<1.0.0,>=0.5.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiobotocore==2.15.2->aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (0.12.0)\n",
      "Requirement already satisfied: boto3<1.35.37,>=1.35.16 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (1.35.36)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (2.4.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (1.5.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (6.1.0)\n",
      "Requirement already satisfied: propcache>=0.2.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (0.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from aiohttp->lightrag-hku==1.0.1) (1.17.1)\n",
      "Requirement already satisfied: POT<0.10,>=0.9 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (0.9.5)\n",
      "Requirement already satisfied: anytree<3.0.0,>=2.12.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (2.12.1)\n",
      "Requirement already satisfied: beartype<0.19.0,>=0.18.5 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (0.18.5)\n",
      "Requirement already satisfied: gensim<5.0.0,>=4.3.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (4.3.3)\n",
      "Requirement already satisfied: graspologic-native<2.0.0,>=1.2.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (1.2.1)\n",
      "Requirement already satisfied: hyppo<0.5.0,>=0.4.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (0.4.0)\n",
      "Requirement already satisfied: joblib<2.0.0,>=1.4.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (1.4.2)\n",
      "Requirement already satisfied: matplotlib<4.0.0,>=3.8.4 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (3.9.2)\n",
      "Requirement already satisfied: scikit-learn<2.0.0,>=1.4.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (1.5.2)\n",
      "Requirement already satisfied: scipy==1.12.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (1.12.0)\n",
      "Requirement already satisfied: seaborn<0.14.0,>=0.13.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (0.13.2)\n",
      "Requirement already satisfied: statsmodels<0.15.0,>=0.14.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (0.14.4)\n",
      "Requirement already satisfied: umap-learn<0.6.0,>=0.5.6 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from graspologic->lightrag-hku==1.0.1) (0.5.7)\n",
      "Requirement already satisfied: pytz in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from neo4j->lightrag-hku==1.0.1) (2024.2)\n",
      "Requirement already satisfied: httpx<0.28.0,>=0.27.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from ollama->lightrag-hku==1.0.1) (0.27.2)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.9.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from ollama->lightrag-hku==1.0.1) (2.9.2)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from openai->lightrag-hku==1.0.1) (4.6.2.post1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from openai->lightrag-hku==1.0.1) (1.9.0)\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from openai->lightrag-hku==1.0.1) (0.7.1)\n",
      "Requirement already satisfied: sniffio in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from openai->lightrag-hku==1.0.1) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from openai->lightrag-hku==1.0.1) (4.67.0)\n",
      "Requirement already satisfied: cryptography>=3.2.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from oracledb->lightrag-hku==1.0.1) (43.0.3)\n",
      "Requirement already satisfied: ipython>=5.3.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from pyvis->lightrag-hku==1.0.1) (8.29.0)\n",
      "Requirement already satisfied: jsonpickle>=1.4.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from pyvis->lightrag-hku==1.0.1) (4.0.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from tiktoken->lightrag-hku==1.0.1) (2024.11.6)\n",
      "Requirement already satisfied: requests>=2.26.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from tiktoken->lightrag-hku==1.0.1) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.21,>=0.20 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from transformers->lightrag-hku==1.0.1) (0.20.3)\n",
      "Requirement already satisfied: idna>=2.8 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai->lightrag-hku==1.0.1) (3.10)\n",
      "Requirement already satisfied: six in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from anytree<3.0.0,>=2.12.1->graspologic->lightrag-hku==1.0.1) (1.16.0)\n",
      "Requirement already satisfied: cffi>=1.12 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from cryptography>=3.2.1->oracledb->lightrag-hku==1.0.1) (1.17.1)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from gensim<5.0.0,>=4.3.2->graspologic->lightrag-hku==1.0.1) (7.0.5)\n",
      "Requirement already satisfied: certifi in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from httpx<0.28.0,>=0.27.0->ollama->lightrag-hku==1.0.1) (2024.8.30)\n",
      "Requirement already satisfied: httpcore==1.* in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from httpx<0.28.0,>=0.27.0->ollama->lightrag-hku==1.0.1) (1.0.6)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.27.0->ollama->lightrag-hku==1.0.1) (0.14.0)\n",
      "Requirement already satisfied: numba>=0.46 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from hyppo<0.5.0,>=0.4.0->graspologic->lightrag-hku==1.0.1) (0.60.0)\n",
      "Requirement already satisfied: autograd>=1.3 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from hyppo<0.5.0,>=0.4.0->graspologic->lightrag-hku==1.0.1) (1.7.0)\n",
      "Requirement already satisfied: decorator in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.19.1)\n",
      "Requirement already satisfied: matplotlib-inline in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.1.7)\n",
      "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (3.0.48)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (2.18.0)\n",
      "Requirement already satisfied: stack-data in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.6.3)\n",
      "Requirement already satisfied: traitlets>=5.13.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (5.14.3)\n",
      "Requirement already satisfied: pexpect>4.3 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (4.9.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from jinja2->torch->lightrag-hku==1.0.1) (2.1.5)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (4.54.1)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (1.4.7)\n",
      "Requirement already satisfied: pillow>=8 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (11.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from matplotlib<4.0.0,>=3.8.4->graspologic->lightrag-hku==1.0.1) (2.9.0.post0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.9.0->ollama->lightrag-hku==1.0.1) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.23.4 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.9.0->ollama->lightrag-hku==1.0.1) (2.23.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken->lightrag-hku==1.0.1) (3.4.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken->lightrag-hku==1.0.1) (2.2.3)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from scikit-learn<2.0.0,>=1.4.2->graspologic->lightrag-hku==1.0.1) (3.5.0)\n",
      "Requirement already satisfied: pandas>=1.2 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from seaborn<0.14.0,>=0.13.2->graspologic->lightrag-hku==1.0.1) (2.2.3)\n",
      "Requirement already satisfied: patsy>=0.5.6 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from statsmodels<0.15.0,>=0.14.2->graspologic->lightrag-hku==1.0.1) (1.0.1)\n",
      "Requirement already satisfied: pynndescent>=0.5 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from umap-learn<0.6.0,>=0.5.6->graspologic->lightrag-hku==1.0.1) (0.5.13)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from boto3<1.35.37,>=1.35.16->aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from boto3<1.35.37,>=1.35.16->aiobotocore[boto3]==2.15.2->aioboto3->lightrag-hku==1.0.1) (0.10.4)\n",
      "Requirement already satisfied: pycparser in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=3.2.1->oracledb->lightrag-hku==1.0.1) (2.22)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from jedi>=0.16->ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.8.4)\n",
      "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from numba>=0.46->hyppo<0.5.0,>=0.4.0->graspologic->lightrag-hku==1.0.1) (0.43.0)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages (from pandas>=1.2->seaborn<0.14.0,>=0.13.2->graspologic->lightrag-hku==1.0.1) (2024.2)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from pexpect>4.3->ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.2.13)\n",
      "Requirement already satisfied: executing>=1.2.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from stack-data->ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (2.1.0)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from stack-data->ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (2.4.1)\n",
      "Requirement already satisfied: pure-eval in /Users/mayaraayat/Library/Python/3.11/lib/python/site-packages (from stack-data->ipython>=5.3.0->pyvis->lightrag-hku==1.0.1) (0.2.3)\n",
      "Installing collected packages: lightrag-hku\n",
      "  Attempting uninstall: lightrag-hku\n",
      "    Found existing installation: lightrag-hku 1.0.1\n",
      "    Uninstalling lightrag-hku-1.0.1:\n",
      "      Successfully uninstalled lightrag-hku-1.0.1\n",
      "\u001b[33m  DEPRECATION: Legacy editable install of lightrag-hku==1.0.1 from file:///Users/mayaraayat/Desktop/axa-fall2024-p2/LightRAG (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457\u001b[0m\u001b[33m\n",
      "\u001b[0m  Running setup.py develop for lightrag-hku\n",
      "Successfully installed lightrag-hku-1.0.1\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install -e ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import logging\n",
    "from lightrag import LightRAG, QueryParam\n",
    "from lightrag.llm import ollama_model_complete, ollama_embedding\n",
    "from lightrag.utils import EmbeddingFunc\n",
    "import pdfplumber"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "WORKING_DIR = os.path.join(os.getcwd(), 'data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/mayaraayat/Desktop/axa-fall2024-p2\n"
     ]
    }
   ],
   "source": [
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightrag:Logger initialized for working directory: /Users/mayaraayat/Desktop/axa-fall2024-p2/data\n",
      "INFO:lightrag:Load KV llm_response_cache with 0 data\n",
      "INFO:lightrag:Load KV full_docs with 0 data\n",
      "INFO:lightrag:Load KV text_chunks with 0 data\n",
      "INFO:nano-vectordb:Init {'embedding_dim': 768, 'metric': 'cosine', 'storage_file': '/Users/mayaraayat/Desktop/axa-fall2024-p2/data/vdb_entities.json'} 0 data\n",
      "INFO:nano-vectordb:Init {'embedding_dim': 768, 'metric': 'cosine', 'storage_file': '/Users/mayaraayat/Desktop/axa-fall2024-p2/data/vdb_relationships.json'} 0 data\n",
      "INFO:nano-vectordb:Init {'embedding_dim': 768, 'metric': 'cosine', 'storage_file': '/Users/mayaraayat/Desktop/axa-fall2024-p2/data/vdb_chunks.json'} 0 data\n"
     ]
    }
   ],
   "source": [
    "rag = LightRAG(\n",
    "   working_dir=WORKING_DIR,\n",
    "   chunk_token_size=1200,\n",
    "   llm_model_func=ollama_model_complete,\n",
    "   llm_model_name=\"llama3.1:latest\",\n",
    "   llm_model_max_async=4,\n",
    "   llm_model_max_token_size=32768,\n",
    "   llm_model_kwargs={\"host\": \"http://localhost:11434\", \"options\": {\"num_ctx\": 32768}},\n",
    "   embedding_func=EmbeddingFunc(\n",
    "       embedding_dim=768,\n",
    "       max_token_size=8192,\n",
    "       func=lambda texts: ollama_embedding(texts, embed_model=\"nomic-embed-text\", host=\"http://localhost:11434\"),\n",
    "   ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_paths = ['april2024/' + file for file in os.listdir('april2024')]\n",
    "text = ''\n",
    "for file_path in files_paths:\n",
    "    with open(file_path, 'rb') as f:\n",
    "        text += f.read().decode('utf-8') + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "This event loop is already running",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[24], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mrag\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minsert\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Desktop/axa-fall2024-p2/LightRAG/lightrag/lightrag.py:225\u001b[0m, in \u001b[0;36mLightRAG.insert\u001b[0;34m(self, string_or_strings)\u001b[0m\n\u001b[1;32m    223\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minsert\u001b[39m(\u001b[38;5;28mself\u001b[39m, string_or_strings):\n\u001b[1;32m    224\u001b[0m     loop \u001b[38;5;241m=\u001b[39m always_get_an_event_loop()\n\u001b[0;32m--> 225\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_until_complete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mainsert\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstring_or_strings\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py:629\u001b[0m, in \u001b[0;36mBaseEventLoop.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m    618\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Run until the Future is done.\u001b[39;00m\n\u001b[1;32m    619\u001b[0m \n\u001b[1;32m    620\u001b[0m \u001b[38;5;124;03mIf the argument is a coroutine, it is wrapped in a Task.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    626\u001b[0m \u001b[38;5;124;03mReturn the Future's result, or raise its exception.\u001b[39;00m\n\u001b[1;32m    627\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    628\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_closed()\n\u001b[0;32m--> 629\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_running\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    631\u001b[0m new_task \u001b[38;5;241m=\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m futures\u001b[38;5;241m.\u001b[39misfuture(future)\n\u001b[1;32m    632\u001b[0m future \u001b[38;5;241m=\u001b[39m tasks\u001b[38;5;241m.\u001b[39mensure_future(future, loop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n",
      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py:588\u001b[0m, in \u001b[0;36mBaseEventLoop._check_running\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    586\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_check_running\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    587\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_running():\n\u001b[0;32m--> 588\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThis event loop is already running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    589\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m events\u001b[38;5;241m.\u001b[39m_get_running_loop() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    590\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m    591\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCannot run the event loop while another loop is running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: This event loop is already running"
     ]
    }
   ],
   "source": [
    "rag.insert(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}