clean rag training

src/model/initialize_deepseek_model.py

@@ -0,0 +1,54 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, BitsAndBytesConfig, EarlyStoppingCallback, PreTrainedTokenizer
+from peft import LoraConfig, get_peft_model, TaskType
+import torch
+
+def initialize_deepseek_model(model, device, tokenizer, train_dataset, val_dataset, MODEL_DIR):
+    lora_config = LoraConfig(
+        r=16,  # Rank of LoRA matrices (adjust for memory vs. accuracy)
+        lora_alpha=32,  # Scaling factor
+        lora_dropout=0.0,  # Dropout for regularization
+        bias="none",
+        task_type=TaskType.CAUSAL_LM,
+        target_modules=[
+            "q_proj",
+            "k_proj",
+            "v_proj",
+            "o_proj",
+            "gate_proj",
+            "up_proj",
+            "down_proj"
+        ]
+    )
+
+    model = get_peft_model(model, lora_config)
+    model = model.to(device)
+
+    training_args = TrainingArguments(
+        output_dir=MODEL_DIR,
+        eval_strategy="epoch",  # Evaluate at the end of each epoch
+        save_strategy="epoch",  # Save model every epoch
+        per_device_train_batch_size=1,  # LoRA allows higher batch size
+        per_device_eval_batch_size=1,
+        gradient_accumulation_steps=16,
+        num_train_epochs=10,  # Increase if needed
+        learning_rate=5e-5,  # Higher LR since we're only training LoRA layers
+        weight_decay=0.001,
+        logging_steps=50,  # Print loss every 50 steps
+        save_total_limit=2,  # Keep last 4 checkpoints
+        bf16=True if torch.cuda.is_available() else False,
+        push_to_hub=False,
+        load_best_model_at_end=True,
+        metric_for_best_model="eval_loss",
+        greater_is_better=False
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=val_dataset,
+        tokenizer=tokenizer,
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=2)]
+    )
+
+    return model, trainer

src/prompts/pre_rag_prompt.py

@@ -1,8 +1,6 @@
-input_text = """You are an assistant that given natural language queries tells me which tables
-are relevant to the query. Here is the schema of the tables of the database.
+input_text = """Given natural language queries tells me which tables
+are relevant to the query. Here is the schema of the database.
 
-Table "team"
-Stores information about NBA teams.
 Table "team" 
 identifier for the team, 
 full official name of the team ("Los Angeles Lakers"),
@@ -38,7 +36,7 @@ assists by the home team,
 steals by the home team,
 blocks by the home team,
 turnovers by the home team,
-personal fouls by the home team,,
+personal fouls by the home team,
 total points scored by the home team,
 plus/minus rating for the home team,
 iD of the away team,
@@ -113,10 +111,6 @@ Response:
 SQLite:
 team
 
-Request:
-"Which team had the highest number of team turnovers in an away game?"
-Response:
-other_stats
 
 Request:
 "What was the average number of fastbreak points scored by the Los Angeles Lakers in home wins during the 2020 season?"

src/rag/get_tokenized_dataset.py

@@ -0,0 +1,47 @@
+from datasets import Dataset
+from sql_metadata import Parser
+
+
+def format_deepseek_chat(example, tokenizer, input_prompt):
+    # Manually build the prompt as one flat string
+    prompt = f"{input_prompt}{example['natural_query']}\n"
+    completion = f"Tables:\n{example['tables']}"
+
+    full_text = prompt + completion
+    tokenized = tokenizer(
+        full_text,
+        truncation=True,
+        padding="max_length",
+        max_length=3156,  # or whatever your model can handle
+    )
+
+    # Mask out prompt tokens in the labels
+    prompt_len = len(tokenizer(prompt, truncation=True)["input_ids"])
+    labels = tokenized["input_ids"][:]
+    labels[:prompt_len] = [-100] * prompt_len
+    tokenized["labels"] = labels
+
+    return tokenized
+
+
+def get_tokenized_dataset(nba_df, tokenizer, input_prompt):
+    natural_query_list = nba_df["natural_query"].tolist()
+    sql_query_list = nba_df["sql_query"].tolist()
+    tables = [Parser(sql_query).tables for sql_query in sql_query_list]
+
+    dataset_dict = {
+        "natural_query": natural_query_list,
+        "tables": tables,
+    }
+
+    # Create HuggingFace Dataset
+    dataset = Dataset.from_dict(dataset_dict)
+
+    tokenized_dataset = dataset.map(
+    lambda x: format_deepseek_chat(x, tokenizer, input_prompt),
+    remove_columns=["natural_query", "tables"]
+    )   
+    split = int(0.9 * len(tokenized_dataset))  # 90% train, 10% validation
+    train_dataset = tokenized_dataset.select(range(split))
+    val_dataset = tokenized_dataset.select(range(split, len(tokenized_dataset)))
+    return train_dataset, val_dataset

train_dynamic_rag.ipynb

@@ -7,7 +7,16 @@
       "metadata": {
         "id": "a87fe5f3"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/opt/anaconda3/envs/CSCI544/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+            "  from .autonotebook import tqdm as notebook_tqdm\n"
+          ]
+        }
+      ],
       "source": [
         "import pandas as pd\n",
         "import torch\n",
@@ -23,9 +32,7 @@
         "import os\n",
         "import numpy as np\n",
         "\n",
-        "with contextlib.redirect_stdout(sys.__stdout__), contextlib.redirect_stderr(sys.__stderr__):\n",
-        "    %pip install datasets\n",
-        "    %pip install sql_metadata\n",
+        "\n",
         "\"\"\"\"\n",
         "with contextlib.redirect_stdout(sys.__stdout__), contextlib.redirect_stderr(sys.__stderr__):\n",
         "    %pip install datasets\n",
@@ -44,7 +51,7 @@
       },
       "outputs": [],
       "source": [
-        "is_google_colab = True\n",
+        "is_google_colab = False\n",
         "use_bnb = False"
       ]
     },
@@ -73,41 +80,7 @@
         "id": "47577a7f",
         "outputId": "999c4e88-3f89-49b1-9e21-abac91703bf3"
       },
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
-          ]
-        },
-        {
-          "output_type": "stream",
-          "name": "stderr",
-          "text": [
-            "/usr/local/lib/python3.11/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n",
-            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
-            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
-            "You will be able to reuse this secret in all of your notebooks.\n",
-            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
-            "  warnings.warn(\n"
-          ]
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "Fetching 37 files:   0%|          | 0/37 [00:00<?, ?it/s]"
-            ],
-            "application/vnd.jupyter.widget-view+json": {
-              "version_major": 2,
-              "version_minor": 0,
-              "model_id": "9200f1303f124bddaa6114cdf0f5f878"
-            }
-          },
-          "metadata": {}
-        }
-      ],
+      "outputs": [],
       "source": [
         "current_read_path = \"./\"\n",
         "current_write_path = \"./\"\n",
@@ -147,6 +120,12 @@
         "from src.prompts.pre_rag_prompt import input_text as input_prompt"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "id": "fdd54aab",
+      "metadata": {},
+      "source": []
+    },
     {
       "cell_type": "code",
       "execution_count": 5,
@@ -183,183 +162,79 @@
       },
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stderr",
-          "text": [
-            "<ipython-input-6-ac29fbf828da>:2: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
-            "  df = df.applymap(lambda x: re.sub(r'\\s+', ' ', x) if isinstance(x, str) else x)\n"
-          ]
-        },
-        {
           "output_type": "stream",
-          "name": "stdout",
           "text": [
-            "Total dataset examples: 1044\n"
+            "/var/folders/g0/47tr69v179dg7w6zyphp9b280000gn/T/ipykernel_70691/3415966509.py:2: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+            "  df = df.applymap(lambda x: re.sub(r'\\s+', ' ', x) if isinstance(x, str) else x)\n"
           ]
         }
       ],
       "source": [
-        "\n",
         "df = pd.read_csv(read_path(\"train-data/sql_train.tsv\"), sep='\\t')\n",
         "df = df.applymap(lambda x: re.sub(r'\\s+', ' ', x) if isinstance(x, str) else x)\n",
         "\n",
-        "# Display dataset info\n",
-        "print(f\"Total dataset examples: {len(df)}\")\n",
-        "\n",
-        "# Load tokenizer\n",
         "model_name = read_path(\"deepseek-coder-1.3b-instruct\")\n",
         "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
-        "\n",
-        "# Enable 8-bit quantization for lower memory usage\n",
-        "bnb_config = None\n",
-        "if use_bnb:\n",
-        "    bnb_config = BitsAndBytesConfig(\n",
-        "        load_in_8bit=True,\n",
-        "        bnb_8bit_compute_dtype=torch.float16\n",
-        "    )\n",
-        "\n",
-        "# Load model with quantization\n",
-        "#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
         "device_name = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
         "device = torch.device(device_name)\n",
+        "\n",
         "model = AutoModelForCausalLM.from_pretrained(\n",
         "    model_name,\n",
-        "    quantization_config=bnb_config,\n",
         "    device_map=device\n",
         ")\n",
-        "\n",
         "tokenizer.truncation_side = \"left\"\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": 7,
-      "id": "7f8b1acf",
-      "metadata": {
-        "id": "7f8b1acf"
-      },
-      "outputs": [],
-      "source": [
-        "natural_query_list = df[\"natural_query\"].tolist()\n",
-        "sql_query_list = df[\"sql_query\"].tolist()\n",
-        "tables = [Parser(sql_query).tables for sql_query in sql_query_list]\n",
-        "\n",
-        "dataset_dict = {\n",
-        "    \"natural_query\": natural_query_list,\n",
-        "    \"tables\": tables,\n",
-        "}\n",
-        "\n",
-        "# Create HuggingFace Dataset\n",
-        "dataset = Dataset.from_dict(dataset_dict)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 8,
-      "id": "f385a9df",
-      "metadata": {
-        "id": "f385a9df"
-      },
-      "outputs": [],
+      "id": "11a687f7",
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Map: 100%|██████████| 1044/1044 [00:07<00:00, 132.69 examples/s]\n"
+          ]
+        }
+      ],
       "source": [
+        "from src.rag.get_tokenized_dataset import get_tokenized_dataset\n",
         "\n",
-        "def format_deepseek_chat(example, tokenizer):\n",
-        "    # Manually build the prompt as one flat string\n",
-        "    prompt = f\"{input_prompt}{example['natural_query']}\\n\"\n",
-        "    completion = f\"Tables:\\n{example['tables']}\"\n",
-        "\n",
-        "    full_text = prompt + completion\n",
-        "    tokenized = tokenizer(\n",
-        "        full_text,\n",
-        "        truncation=True,\n",
-        "        padding=\"max_length\",\n",
-        "        max_length=3156,  # or whatever your model can handle\n",
-        "    )\n",
-        "\n",
-        "    # Mask out prompt tokens in the labels\n",
-        "    prompt_len = len(tokenizer(prompt, truncation=True)[\"input_ids\"])\n",
-        "    labels = tokenized[\"input_ids\"][:]\n",
-        "    labels[:prompt_len] = [-100] * prompt_len\n",
-        "    tokenized[\"labels\"] = labels\n",
-        "\n",
-        "    return tokenized\n"
+        "train_dataset, val_dataset = get_tokenized_dataset(df, tokenizer, input_prompt)\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": 9,
-      "id": "43562f78",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 121,
-          "referenced_widgets": [
-            "68ff2fc00bd041e7b79a811e3de1e596",
-            "4c41e81bcd254df7b1265206a5a6b40b",
-            "1a8c093fccbb437db6e0390a920f5cc5",
-            "e11d04a9d22a4229922e3eb4e3eb6466",
-            "5d89a5574a3d4a8993e6dca78d406d2d",
-            "dd24270dc07942a6972fbfaf58129989",
-            "643903cd7a5b4a52a4687ec38eb8c4dc",
-            "13ae11c314664c44ae18d35cf57a1334",
-            "e68cfd05ba994a34b93107d2eab82ad3",
-            "ea283e7e8b234519b881c562b7eb01d3",
-            "1ec5329ea0434df4b74d0f311e016c3e"
-          ]
-        },
-        "id": "43562f78",
-        "outputId": "58e8ce3f-b7cd-4cf6-dfa4-180b4a699cf9"
-      },
+      "id": "bc8b8212",
+      "metadata": {},
       "outputs": [
         {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "Map:   0%|          | 0/1044 [00:00<?, ? examples/s]"
-            ],
-            "application/vnd.jupyter.widget-view+json": {
-              "version_major": 2,
-              "version_minor": 0,
-              "model_id": "68ff2fc00bd041e7b79a811e3de1e596"
-            }
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "939\n",
-            "105\n",
-            "{'input_ids': [32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32013, 2042, 417, 274, 20391, 344, 2017, 3881, 4694, 12780, 10510, 523, 585, 7214, 185, 554, 7688, 276, 254, 5151, 13, 4451, 317, 254, 16135, 280, 254, 7214, 280, 254, 4892, 13, 185, 185, 5127, 440, 21598, 1, 185, 50, 577, 379, 1748, 782, 461, 8443, 9474, 13, 185, 5127, 440, 21598, 1, 207, 185, 19655, 327, 254, 2547, 11, 207, 185, 9875, 6270, 1208, 280, 254, 2547, 8507, 43, 378, 14204, 412, 9961, 8472, 185, 11972, 2942, 2547, 1208, 8507, 43, 1743, 8472, 185, 77, 767, 1523, 327, 254, 2547, 8507, 43, 9961, 8472, 185, 23861, 1064, 254, 2547, 317, 2842, 11, 185, 4968, 1064, 254, 2547, 317, 6288, 11, 185, 5456, 254, 2547, 438, 8143, 185, 477, 185, 185, 5127, 440, 14641, 1, 185, 21810, 21411, 11, 31131, 372, 440, 17, 19393, 19393, 1, 8507, 17, 16, 24, 22, 15, 1, 327, 254, 207, 16, 24, 22, 15, 4314, 8, 185, 19655, 280, 254, 1712, 2547, 11, 207, 185, 356, 26321, 335, 280, 254, 1712, 2547, 11, 185, 9875, 1208, 280, 254, 1712, 2547, 11, 185, 19464, 21411, 327, 254, 2612, 11, 185, 1984, 254, 2612, 438, 7226, 334, 19393, 19393, 12, 8213, 12, 7127, 650, 185, 1, 54, 1, 562, 254, 1712, 2547, 2103, 11, 440, 43, 1, 562, 653, 4726, 11, 185, 11695, 4054, 7226, 279, 254, 2612, 11, 185, 3267, 9054, 1396, 457, 254, 1712, 2547, 11, 185, 3267, 9054, 18012, 457, 254, 1712, 2547, 11, 185, 3267, 6206, 14986, 280, 254, 1712, 2547, 11, 185, 14565, 12, 3772, 2010, 9054, 1396, 457, 254, 1712, 2547, 11, 185, 14565, 12, 3772, 15343, 457, 254, 1712, 2547, 11, 185, 14565, 12, 3772, 2010, 6206, 14986, 280, 254, 1712, 2547, 11, 185, 6630, 8474, 1396, 457, 254, 1712, 2547, 11, 185, 6630, 8474, 18012, 457, 254, 1712, 2547, 11, 185, 6630, 5245, 14986, 280, 254, 1712, 2547, 11, 185, 2959, 4630, 11435, 5740, 457, 254, 1712, 2547, 11, 185, 1551, 4630, 11435, 5740, 457, 254, 1712, 2547, 11, 185, 11695, 11435, 5740, 457, 254, 1712, 2547, 11, 185, 468, 1923, 457, 254, 1712, 2547, 11, 185, 7537, 909, 457, 254, 1712, 2547, 11, 185, 28835, 457, 254, 1712, 2547, 11, 185, 788, 17396, 457, 254, 1712, 2547, 11, 185, 28200, 3931, 2724, 457, 254, 1712, 2547, 19555, 185, 11695, 3472, 18605, 457, 254, 1712, 2547, 11, 185, 13289, 14, 10646, 14026, 327, 254, 1712, 2547, 11, 185, 72, 35, 280, 254, 2292, 2547, 11, 185, 356, 26321, 335, 280, 254, 2292, 2547, 11, 185, 9875, 1208, 280, 254, 2292, 2547, 11, 185, 10108, 393, 4283, 473, 254, 2292, 2547, 486, 82, 12422, 11, 185, 1, 54, 1, 562, 254, 2292, 2547, 2103, 11, 440, 43, 1, 562, 653, 4726, 11, 185, 3267, 9054, 1396, 457, 254, 2292, 2547, 11, 185, 3267, 9054, 18012, 457, 254, 2292, 2547, 11, 185, 3267, 6206, 14986, 280, 254, 2292, 2547, 11, 185, 14565, 12, 3772, 2010, 9054, 1396, 457, 254, 2292, 2547, 11, 185, 14565, 12, 3772, 15343, 457, 254, 2292, 2547, 11, 185, 14565, 12, 3772, 2010, 6206, 14986, 280, 254, 2292, 2547, 11, 185, 6630, 8474, 1396, 457, 254, 2292, 2547, 11, 185, 6630, 8474, 18012, 457, 254, 2292, 2547, 11, 185, 6630, 5245, 14986, 280, 254, 2292, 2547, 11, 185, 2959, 4630, 11435, 5740, 457, 254, 2292, 2547, 11, 185, 1551, 4630, 11435, 5740, 457, 254, 2292, 2547, 11, 185, 11695, 11435, 5740, 457, 254, 2292, 2547, 11, 185, 468, 1923, 457, 254, 2292, 2547, 11, 185, 7537, 909, 457, 254, 2292, 2547, 11, 185, 28835, 457, 254, 2292, 2547, 11, 185, 788, 17396, 457, 254, 2292, 2547, 11, 185, 28200, 3931, 2724, 457, 254, 2292, 2547, 11, 185, 11695, 3472, 18605, 457, 254, 2292, 2547, 11, 185, 13289, 14, 10646, 14026, 327, 254, 2292, 2547, 11, 185, 15367, 980, 3192, 3905, 317, 2315, 334, 16, 405, 7589, 11, 207, 15, 405, 2357, 650, 185, 13388, 4314, 409, 1530, 23836, 11, 185, 477, 185, 185, 5127, 440, 1156, 62, 16204, 1, 185, 50, 577, 379, 4577, 13024, 11, 12144, 276, 254, 2612, 2365, 3752, 2612, 62, 304, 13, 185, 13403, 11866, 15787, 5787, 7449, 30862, 440, 1156, 62, 16204, 1, 334, 185, 19464, 2612, 21411, 11, 12050, 1975, 3812, 473, 2612, 2365, 185, 275, 6006, 21411, 185, 5816, 2547, 21411, 185, 5816, 2547, 31593, 335, 185, 5816, 2547, 3775, 185, 12168, 279, 254, 7416, 457, 254, 1712, 2547, 185, 9353, 5504, 3472, 457, 254, 1712, 2547, 185, 7212, 2963, 3472, 457, 254, 1712, 2547, 185, 17819, 370, 2012, 457, 254, 1712, 2547, 185, 7675, 280, 2012, 4177, 207, 185, 7675, 280, 2591, 254, 8129, 438, 16538, 185, 5816, 2547, 1936, 17396, 185, 11695, 1936, 17396, 457, 254, 1712, 2547, 185, 5816, 2547, 11435, 5740, 185, 12168, 838, 1936, 17396, 457, 254, 1712, 2547, 185, 11507, 2547, 21411, 185, 11507, 2547, 31593, 335, 185, 12168, 279, 254, 7416, 457, 254, 2292, 2547, 185, 9353, 5504, 3472, 457, 254, 2292, 2547, 185, 7212, 2963, 3472, 457, 254, 2292, 2547, 185, 17819, 370, 2012, 457, 254, 2292, 2547, 185, 11507, 2547, 1936, 17396, 185, 11695, 1936, 17396, 457, 254, 2292, 2547, 185, 11507, 2547, 11435, 5740, 185, 12168, 838, 1936, 17396, 457, 254, 2292, 2547, 185, 477, 185, 185, 185, 7605, 387, 885, 254, 4761, 280, 254, 2365, 344, 417, 4362, 276, 3495, 254, 3881, 4694, 5151, 11, 14843, 457, 929, 281, 11, 533, 441, 2816, 274, 11543, 13, 185, 185, 1459, 2194, 11, 185, 6522, 25, 185, 1, 2628, 317, 254, 1093, 3472, 254, 10851, 14204, 412, 9961, 463, 2634, 18605, 429, 1712, 1956, 185, 6522, 25, 185, 14641, 185, 185, 6522, 25, 185, 1, 15575, 9474, 417, 6288, 279, 254, 1967, 280, 8700, 1956, 185, 6231, 547, 25, 185, 21598, 185, 185, 4397, 25, 185, 1, 15575, 2547, 658, 254, 7495, 1594, 280, 2547, 1936, 17396, 279, 274, 2292, 2612, 1956, 185, 6522, 25, 185, 1156, 62, 16204, 185, 185, 4397, 25, 185, 1, 2628, 438, 254, 5126, 1594, 280, 4299, 9351, 3472, 18605, 457, 254, 10851, 14204, 412, 9961, 279, 1712, 19998, 2310, 254, 207, 17, 15, 17, 15, 4314, 1956, 185, 6522, 25, 185, 14641, 11, 746, 62, 16204, 185, 185, 4888, 317, 254, 3092, 25, 185, 2808, 1311, 3212, 3472, 1213, 254, 11738, 21915, 82, 8129, 2310, 254, 207, 16, 24, 24, 21, 4314, 30, 185, 51, 2368, 25, 185, 3204, 14641, 3676], 'attention_mask': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32014, 32013, 2042, 417, 274, 20391, 344, 2017, 3881, 4694, 12780, 10510, 523, 585, 7214, 185, 554, 7688, 276, 254, 5151, 13, 4451, 317, 254, 16135, 280, 254, 7214, 280, 254, 4892, 13, 185, 185, 5127, 440, 21598, 1, 185, 50, 577, 379, 1748, 782, 461, 8443, 9474, 13, 185, 5127, 440, 21598, 1, 207, 185, 19655, 327, 254, 2547, 11, 207, 185, 9875, 6270, 1208, 280, 254, 2547, 8507, 43, 378, 14204, 412, 9961, 8472, 185, 11972, 2942, 2547, 1208, 8507, 43, 1743, 8472, 185, 77, 767, 1523, 327, 254, 2547, 8507, 43, 9961, 8472, 185, 23861, 1064, 254, 2547, 317, 2842, 11, 185, 4968, 1064, 254, 2547, 317, 6288, 11, 185, 5456, 254, 2547, 438, 8143, 185, 477, 185, 185, 5127, 440, 14641, 1, 185, 21810, 21411, 11, 31131, 372, 440, 17, 19393, 19393, 1, 8507, 17, 16, 24, 22, 15, 1, 327, 254, 207, 16, 24, 22, 15, 4314, 8, 185, 19655, 280, 254, 1712, 2547, 11, 207, 185, 356, 26321, 335, 280, 254, 1712, 2547, 11, 185, 9875, 1208, 280, 254, 1712, 2547, 11, 185, 19464, 21411, 327, 254, 2612, 11, 185, 1984, 254, 2612, 438, 7226, 334, 19393, 19393, 12, 8213, 12, 7127, 650, 185, 1, 54, 1, 562, 254, 1712, 2547, 2103, 11, 440, 43, 1, 562, 653, 4726, 11, 185, 11695, 4054, 7226, 279, 254, 2612, 11, 185, 3267, 9054, 1396, 457, 254, 1712, 2547, 11, 185, 3267, 9054, 18012, 457, 254, 1712, 2547, 11, 185, 3267, 6206, 14986, 280, 254, 1712, 2547, 11, 185, 14565, 12, 3772, 2010, 9054, 1396, 457, 254, 1712, 2547, 11, 185, 14565, 12, 3772, 15343, 457, 254, 1712, 2547, 11, 185, 14565, 12, 3772, 2010, 6206, 14986, 280, 254, 1712, 2547, 11, 185, 6630, 8474, 1396, 457, 254, 1712, 2547, 11, 185, 6630, 8474, 18012, 457, 254, 1712, 2547, 11, 185, 6630, 5245, 14986, 280, 254, 1712, 2547, 11, 185, 2959, 4630, 11435, 5740, 457, 254, 1712, 2547, 11, 185, 1551, 4630, 11435, 5740, 457, 254, 1712, 2547, 11, 185, 11695, 11435, 5740, 457, 254, 1712, 2547, 11, 185, 468, 1923, 457, 254, 1712, 2547, 11, 185, 7537, 909, 457, 254, 1712, 2547, 11, 185, 28835, 457, 254, 1712, 2547, 11, 185, 788, 17396, 457, 254, 1712, 2547, 11, 185, 28200, 3931, 2724, 457, 254, 1712, 2547, 19555, 185, 11695, 3472, 18605, 457, 254, 1712, 2547, 11, 185, 13289, 14, 10646, 14026, 327, 254, 1712, 2547, 11, 185, 72, 35, 280, 254, 2292, 2547, 11, 185, 356, 26321, 335, 280, 254, 2292, 2547, 11, 185, 9875, 1208, 280, 254, 2292, 2547, 11, 185, 10108, 393, 4283, 473, 254, 2292, 2547, 486, 82, 12422, 11, 185, 1, 54, 1, 562, 254, 2292, 2547, 2103, 11, 440, 43, 1, 562, 653, 4726, 11, 185, 3267, 9054, 1396, 457, 254, 2292, 2547, 11, 185, 3267, 9054, 18012, 457, 254, 2292, 2547, 11, 185, 3267, 6206, 14986, 280, 254, 2292, 2547, 11, 185, 14565, 12, 3772, 2010, 9054, 1396, 457, 254, 2292, 2547, 11, 185, 14565, 12, 3772, 15343, 457, 254, 2292, 2547, 11, 185, 14565, 12, 3772, 2010, 6206, 14986, 280, 254, 2292, 2547, 11, 185, 6630, 8474, 1396, 457, 254, 2292, 2547, 11, 185, 6630, 8474, 18012, 457, 254, 2292, 2547, 11, 185, 6630, 5245, 14986, 280, 254, 2292, 2547, 11, 185, 2959, 4630, 11435, 5740, 457, 254, 2292, 2547, 11, 185, 1551, 4630, 11435, 5740, 457, 254, 2292, 2547, 11, 185, 11695, 11435, 5740, 457, 254, 2292, 2547, 11, 185, 468, 1923, 457, 254, 2292, 2547, 11, 185, 7537, 909, 457, 254, 2292, 2547, 11, 185, 28835, 457, 254, 2292, 2547, 11, 185, 788, 17396, 457, 254, 2292, 2547, 11, 185, 28200, 3931, 2724, 457, 254, 2292, 2547, 11, 185, 11695, 3472, 18605, 457, 254, 2292, 2547, 11, 185, 13289, 14, 10646, 14026, 327, 254, 2292, 2547, 11, 185, 15367, 980, 3192, 3905, 317, 2315, 334, 16, 405, 7589, 11, 207, 15, 405, 2357, 650, 185, 13388, 4314, 409, 1530, 23836, 11, 185, 477, 185, 185, 5127, 440, 1156, 62, 16204, 1, 185, 50, 577, 379, 4577, 13024, 11, 12144, 276, 254, 2612, 2365, 3752, 2612, 62, 304, 13, 185, 13403, 11866, 15787, 5787, 7449, 30862, 440, 1156, 62, 16204, 1, 334, 185, 19464, 2612, 21411, 11, 12050, 1975, 3812, 473, 2612, 2365, 185, 275, 6006, 21411, 185, 5816, 2547, 21411, 185, 5816, 2547, 31593, 335, 185, 5816, 2547, 3775, 185, 12168, 279, 254, 7416, 457, 254, 1712, 2547, 185, 9353, 5504, 3472, 457, 254, 1712, 2547, 185, 7212, 2963, 3472, 457, 254, 1712, 2547, 185, 17819, 370, 2012, 457, 254, 1712, 2547, 185, 7675, 280, 2012, 4177, 207, 185, 7675, 280, 2591, 254, 8129, 438, 16538, 185, 5816, 2547, 1936, 17396, 185, 11695, 1936, 17396, 457, 254, 1712, 2547, 185, 5816, 2547, 11435, 5740, 185, 12168, 838, 1936, 17396, 457, 254, 1712, 2547, 185, 11507, 2547, 21411, 185, 11507, 2547, 31593, 335, 185, 12168, 279, 254, 7416, 457, 254, 2292, 2547, 185, 9353, 5504, 3472, 457, 254, 2292, 2547, 185, 7212, 2963, 3472, 457, 254, 2292, 2547, 185, 17819, 370, 2012, 457, 254, 2292, 2547, 185, 11507, 2547, 1936, 17396, 185, 11695, 1936, 17396, 457, 254, 2292, 2547, 185, 11507, 2547, 11435, 5740, 185, 12168, 838, 1936, 17396, 457, 254, 2292, 2547, 185, 477, 185, 185, 185, 7605, 387, 885, 254, 4761, 280, 254, 2365, 344, 417, 4362, 276, 3495, 254, 3881, 4694, 5151, 11, 14843, 457, 929, 281, 11, 533, 441, 2816, 274, 11543, 13, 185, 185, 1459, 2194, 11, 185, 6522, 25, 185, 1, 2628, 317, 254, 1093, 3472, 254, 10851, 14204, 412, 9961, 463, 2634, 18605, 429, 1712, 1956, 185, 6522, 25, 185, 14641, 185, 185, 6522, 25, 185, 1, 15575, 9474, 417, 6288, 279, 254, 1967, 280, 8700, 1956, 185, 6231, 547, 25, 185, 21598, 185, 185, 4397, 25, 185, 1, 15575, 2547, 658, 254, 7495, 1594, 280, 2547, 1936, 17396, 279, 274, 2292, 2612, 1956, 185, 6522, 25, 185, 1156, 62, 16204, 185, 185, 4397, 25, 185, 1, 2628, 438, 254, 5126, 1594, 280, 4299, 9351, 3472, 18605, 457, 254, 10851, 14204, 412, 9961, 279, 1712, 19998, 2310, 254, 207, 17, 15, 17, 15, 4314, 1956, 185, 6522, 25, 185, 14641, 11, 746, 62, 16204, 185, 185, 4888, 317, 254, 3092, 25, 185, 2808, 1311, 3212, 3472, 1213, 254, 11738, 21915, 82, 8129, 2310, 254, 207, 16, 24, 24, 21, 4314, 30, 185, 51, 2368, 25, 185, 3204, 14641, 3676]}\n"
+          "ename": "AttributeError",
+          "evalue": "'tuple' object has no attribute '__dict__'",
+          "output_type": "error",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+            "Cell \u001b[0;32mIn[9], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msrc\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01minitialize_deepseek_model\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m initialize_deepseek_model\n\u001b[0;32m----> 3\u001b[0m model, trainer \u001b[38;5;241m=\u001b[39m initialize_deepseek_model(model, device, tokenizer, train_dataset, val_dataset, MODEL_DIR)\n",
+            "File \u001b[0;32m~/Documents/USC/spring_2025/NLP/SQL-Generation/src/model/initialize_deepseek_model.py:23\u001b[0m, in \u001b[0;36minitialize_deepseek_model\u001b[0;34m(model, device, tokenizer, train_dataset, val_dataset, MODEL_DIR)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minitialize_deepseek_model\u001b[39m(model, device, tokenizer, train_dataset, val_dataset, MODEL_DIR):\n\u001b[1;32m      6\u001b[0m     lora_config \u001b[38;5;241m=\u001b[39m LoraConfig(\n\u001b[1;32m      7\u001b[0m         r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m16\u001b[39m,  \u001b[38;5;66;03m# Rank of LoRA matrices (adjust for memory vs. accuracy)\u001b[39;00m\n\u001b[1;32m      8\u001b[0m         lora_alpha\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m32\u001b[39m,  \u001b[38;5;66;03m# Scaling factor\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     20\u001b[0m         ]\n\u001b[1;32m     21\u001b[0m     )\n\u001b[0;32m---> 23\u001b[0m     model \u001b[38;5;241m=\u001b[39m get_peft_model(model, lora_config)\n\u001b[1;32m     24\u001b[0m     model \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m     26\u001b[0m     training_args \u001b[38;5;241m=\u001b[39m TrainingArguments(\n\u001b[1;32m     27\u001b[0m         output_dir\u001b[38;5;241m=\u001b[39mMODEL_DIR,\n\u001b[1;32m     28\u001b[0m         eval_strategy\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mepoch\u001b[39m\u001b[38;5;124m\"\u001b[39m,  \u001b[38;5;66;03m# Evaluate at the end of each epoch\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     42\u001b[0m         greater_is_better\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m     43\u001b[0m     )\n",
+            "File \u001b[0;32m/opt/anaconda3/envs/CSCI544/lib/python3.11/site-packages/peft/mapping_func.py:66\u001b[0m, in \u001b[0;36mget_peft_model\u001b[0;34m(model, peft_config, adapter_name, mixed, autocast_adapter_dtype, revision, low_cpu_mem_usage)\u001b[0m\n\u001b[1;32m     64\u001b[0m model_config \u001b[38;5;241m=\u001b[39m BaseTuner\u001b[38;5;241m.\u001b[39mget_model_config(model)\n\u001b[1;32m     65\u001b[0m old_name \u001b[38;5;241m=\u001b[39m peft_config\u001b[38;5;241m.\u001b[39mbase_model_name_or_path\n\u001b[0;32m---> 66\u001b[0m new_name \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname_or_path\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m     67\u001b[0m peft_config\u001b[38;5;241m.\u001b[39mbase_model_name_or_path \u001b[38;5;241m=\u001b[39m new_name\n\u001b[1;32m     69\u001b[0m \u001b[38;5;66;03m# Especially in notebook environments there could be a case that a user wants to experiment with different\u001b[39;00m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;66;03m# configuration values. However, it is likely that there won't be any changes for new configs on an already\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;66;03m# initialized PEFT model. The best we can do is warn the user about it.\u001b[39;00m\n",
+            "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute '__dict__'"
           ]
         }
       ],
       "source": [
+        "from src.model.initialize_deepseek_model import initialize_deepseek_model\n",
         "\n",
-        "\n",
-        "# Apply formatting\n",
-        "tokenized_dataset = dataset.map(\n",
-        "    lambda x: format_deepseek_chat(x, tokenizer),\n",
-        "    remove_columns=[\"natural_query\", \"tables\"]\n",
-        ")\n",
-        "\n",
-        "# Split into train/validation\n",
-        "split = int(0.9 * len(tokenized_dataset))  # 90% train, 10% validation\n",
-        "train_dataset = tokenized_dataset.select(range(split))\n",
-        "val_dataset = tokenized_dataset.select(range(split, len(tokenized_dataset)))\n",
-        "\n",
-        "print(len(train_dataset))\n",
-        "print(len(val_dataset))\n",
-        "\n",
-        "for v in val_dataset:\n",
-        "    print(v)\n",
-        "    break"
+        "model, trainer = initialize_deepseek_model(model, device, tokenizer, train_dataset, val_dataset, MODEL_DIR)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 10,
+      "execution_count": 9,
       "id": "8890a657",
       "metadata": {
         "colab": {
@@ -370,10 +245,18 @@
       },
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "'NoneType' object has no attribute 'cadam32bit_grad_fp32'\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
           "text": [
-            "trainable params: 14,991,360 || all params: 1,361,463,296 || trainable%: 1.1011\n"
+            "/opt/anaconda3/envs/CSCI544/lib/python3.11/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
+            "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n"
           ]
         }
       ],
@@ -398,13 +281,12 @@
         "\n",
         "# Wrap model with LoRA adapters\n",
         "model = get_peft_model(model, lora_config)\n",
-        "model = model.to(device)\n",
-        "model.print_trainable_parameters()  # Show trainable parameters count"
+        "model = model.to(device)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 11,
+      "execution_count": 10,
       "id": "d9508451",
       "metadata": {
         "colab": {
@@ -415,10 +297,10 @@
       },
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stderr",
+          "output_type": "stream",
           "text": [
-            "<ipython-input-11-319f42a4ed7b>:21: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+            "/var/folders/g0/47tr69v179dg7w6zyphp9b280000gn/T/ipykernel_70404/2486149154.py:21: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
             "  trainer = Trainer(\n",
             "No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.\n"
           ]
@@ -457,7 +339,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 11,
       "id": "b0ff5278",
       "metadata": {
         "colab": {
@@ -469,101 +351,13 @@
       },
       "outputs": [
         {
-          "output_type": "stream",
           "name": "stderr",
+          "output_type": "stream",
           "text": [
-            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
-            "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.\n",
-            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mlicesma\u001b[0m (\u001b[33mlicesma-usc\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
+            "\n",
+            "KeyboardInterrupt\n",
+            "\n"
           ]
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
-            "text/html": [
-              "Tracking run with wandb version 0.19.9"
-            ]
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
-            "text/html": [
-              "Run data is saved locally in <code>/content/wandb/run-20250420_174906-5ypbflqe</code>"
-            ]
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
-            "text/html": [
-              "Syncing run <strong><a href='https://wandb.ai/licesma-usc/huggingface/runs/5ypbflqe' target=\"_blank\">/content/drive/MyDrive/sql_gen/dyn_rag_test</a></strong> to <a href='https://wandb.ai/licesma-usc/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"
-            ]
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
-            "text/html": [
-              " View project at <a href='https://wandb.ai/licesma-usc/huggingface' target=\"_blank\">https://wandb.ai/licesma-usc/huggingface</a>"
-            ]
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
-            "text/html": [
-              " View run at <a href='https://wandb.ai/licesma-usc/huggingface/runs/5ypbflqe' target=\"_blank\">https://wandb.ai/licesma-usc/huggingface/runs/5ypbflqe</a>"
-            ]
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "<IPython.core.display.HTML object>"
-            ],
-            "text/html": [
-              "\n",
-              "    <div>\n",
-              "      \n",
-              "      <progress value='4' max='580' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-              "      [  4/580 00:11 < 54:56, 0.17 it/s, Epoch 0.05/10]\n",
-              "    </div>\n",
-              "    <table border=\"1\" class=\"dataframe\">\n",
-              "  <thead>\n",
-              " <tr style=\"text-align: left;\">\n",
-              "      <th>Epoch</th>\n",
-              "      <th>Training Loss</th>\n",
-              "      <th>Validation Loss</th>\n",
-              "    </tr>\n",
-              "  </thead>\n",
-              "  <tbody>\n",
-              "  </tbody>\n",
-              "</table><p>"
-            ]
-          },
-          "metadata": {}
         }
       ],
       "source": [
@@ -578,6 +372,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "id": "J7qO7FE73i40",
+      "metadata": {
+        "id": "J7qO7FE73i40"
+      },
+      "outputs": [],
       "source": [
         "\n",
         "# Prepare query with the same prompt\n",
@@ -593,16 +393,16 @@
         "model_output = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)\n",
         "\n",
         "print(\"Generated Tables:\", model_output)"
-      ],
-      "metadata": {
-        "id": "J7qO7FE73i40"
-      },
-      "id": "J7qO7FE73i40",
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "id": "kwHMVyQa3n89",
+      "metadata": {
+        "id": "kwHMVyQa3n89"
+      },
+      "outputs": [],
       "source": [
         "import sqlite3 as sql\n",
         "\n",
@@ -620,41 +420,40 @@
         "    question, tables = full_example[prompt_length:].split(\"Tables:\\n\")\n",
         "    print(question)\n",
         "    print(tables)\n",
-        "    break\n",
-        ""
-      ],
-      "metadata": {
-        "id": "kwHMVyQa3n89"
-      },
-      "id": "kwHMVyQa3n89",
-      "execution_count": null,
-      "outputs": []
+        "    break\n"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "id": "LhiHqAaB9uE4",
+      "metadata": {
+        "id": "LhiHqAaB9uE4"
+      },
+      "outputs": [],
       "source": [
         "def extract_tables_from_string(s):\n",
         "    keywords = {\"game\", \"team\", \"other_stats\"}\n",
         "    found = {k for k in keywords if k in s}\n",
         "    return found"
-      ],
-      "metadata": {
-        "id": "LhiHqAaB9uE4"
-      },
-      "id": "LhiHqAaB9uE4",
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [],
+      "id": "Kdd8nxWD9txh",
       "metadata": {
         "id": "Kdd8nxWD9txh"
       },
-      "id": "Kdd8nxWD9txh"
+      "source": []
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "id": "KjAXaUgp4TfY",
+      "metadata": {
+        "id": "KjAXaUgp4TfY"
+      },
+      "outputs": [],
       "source": [
         "def compare_table_lists(actual_tables, generated_tables):\n",
         "    actual_set = extract_tables_from_string(actual_tables)\n",
@@ -662,16 +461,16 @@
         "\n",
         "    # Check if they match\n",
         "    return generated_set == actual_set"
-      ],
-      "metadata": {
-        "id": "KjAXaUgp4TfY"
-      },
-      "id": "KjAXaUgp4TfY",
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "id": "8h7bpMML6G6v",
+      "metadata": {
+        "id": "8h7bpMML6G6v"
+      },
+      "outputs": [],
       "source": [
         "\n",
         "num_sql_matched = 0\n",
@@ -709,16 +508,16 @@
         "\n",
         "print(\"Accuracy :\", num_sql_matched/len(val_dataset))\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "8h7bpMML6G6v"
-      },
-      "id": "8h7bpMML6G6v",
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "id": "CoJeZ4FoUMp_",
+      "metadata": {
+        "id": "CoJeZ4FoUMp_"
+      },
+      "outputs": [],
       "source": [
         "\n",
         "num_sql_matched = 0\n",
@@ -756,29 +555,28 @@
         "\n",
         "print(\"Accuracy :\", num_sql_matched/len(val_dataset))\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "CoJeZ4FoUMp_"
-      },
-      "execution_count": null,
-      "outputs": [],
-      "id": "CoJeZ4FoUMp_"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "model = AutoModelForCausalLM.from_pretrained(MODEL_DIR, torch_dtype=torch.bfloat16, device_map=device)\n",
-        "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)\n"
-      ],
+      "execution_count": null,
+      "id": "lNG1joS3T8DN",
       "metadata": {
         "id": "lNG1joS3T8DN"
       },
-      "id": "lNG1joS3T8DN",
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "model = AutoModelForCausalLM.from_pretrained(MODEL_DIR, torch_dtype=torch.bfloat16, device_map=device)\n",
+        "tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)\n"
+      ]
     }
   ],
   "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "A100",
+      "provenance": []
+    },
     "kernelspec": {
       "display_name": "Python 3",
       "name": "python3"
@@ -795,39 +593,64 @@
       "pygments_lexer": "ipython3",
       "version": "3.11.11"
     },
-    "colab": {
-      "provenance": [],
-      "gpuType": "A100"
-    },
-    "accelerator": "GPU",
     "widgets": {
       "application/vnd.jupyter.widget-state+json": {
-        "9200f1303f124bddaa6114cdf0f5f878": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_name": "HBoxModel",
-          "model_module_version": "1.5.0",
+        "13ae11c314664c44ae18d35cf57a1334": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HBoxModel",
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
             "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HBoxView",
-            "box_style": "",
-            "children": [
-              "IPY_MODEL_17ddbb74e1764f37b8d34c311fae200c",
-              "IPY_MODEL_ef732739334b4ac593fd665e01cd83c1",
-              "IPY_MODEL_949ee3d1a9cd4060864dec5d4283ef2c"
-            ],
-            "layout": "IPY_MODEL_b98629e053674527aacca899ab7f11a9"
-          }
-        },
-        "17ddbb74e1764f37b8d34c311fae200c": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_name": "HTMLModel",
-          "model_module_version": "1.5.0",
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "17ddbb74e1764f37b8d34c311fae200c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
           "state": {
             "_dom_classes": [],
             "_model_module": "@jupyter-widgets/controls",
@@ -845,10 +668,10 @@
             "value": "Fetching 37 files: 100%"
           }
         },
-        "ef732739334b4ac593fd665e01cd83c1": {
+        "1a8c093fccbb437db6e0390a920f5cc5": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "FloatProgressModel",
           "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
           "state": {
             "_dom_classes": [],
             "_model_module": "@jupyter-widgets/controls",
@@ -861,39 +684,33 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_3b80c66e0f384c45ab4187301599fab2",
-            "max": 37,
+            "layout": "IPY_MODEL_13ae11c314664c44ae18d35cf57a1334",
+            "max": 1044,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_db6a23e658a34722a8f22505c6ace7b4",
-            "value": 37
+            "style": "IPY_MODEL_e68cfd05ba994a34b93107d2eab82ad3",
+            "value": 1044
           }
         },
-        "949ee3d1a9cd4060864dec5d4283ef2c": {
+        "1ec5329ea0434df4b74d0f311e016c3e": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "HTMLModel",
           "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
           "state": {
-            "_dom_classes": [],
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
+            "_model_name": "DescriptionStyleModel",
             "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_7751defbc4534d518d9e923b9019aa8b",
-            "placeholder": "​",
-            "style": "IPY_MODEL_fe6352bce22a40e7a936e7f90313bd02",
-            "value": " 37/37 [00:00&lt;00:00, 3657.54it/s]"
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
           }
         },
-        "b98629e053674527aacca899ab7f11a9": {
+        "3b80c66e0f384c45ab4187301599fab2": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -942,10 +759,46 @@
             "width": null
           }
         },
-        "84cc47dc70864bf3aa7599c06eb13c51": {
+        "4c41e81bcd254df7b1265206a5a6b40b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_dd24270dc07942a6972fbfaf58129989",
+            "placeholder": "​",
+            "style": "IPY_MODEL_643903cd7a5b4a52a4687ec38eb8c4dc",
+            "value": "Map: 100%"
+          }
+        },
+        "5d711bb927024d8d9f9b8bb685d6f388": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "5d89a5574a3d4a8993e6dca78d406d2d": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -994,10 +847,10 @@
             "width": null
           }
         },
-        "5d711bb927024d8d9f9b8bb685d6f388": {
+        "643903cd7a5b4a52a4687ec38eb8c4dc": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
           "state": {
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
@@ -1009,10 +862,32 @@
             "description_width": ""
           }
         },
-        "3b80c66e0f384c45ab4187301599fab2": {
+        "68ff2fc00bd041e7b79a811e3de1e596": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_4c41e81bcd254df7b1265206a5a6b40b",
+              "IPY_MODEL_1a8c093fccbb437db6e0390a920f5cc5",
+              "IPY_MODEL_e11d04a9d22a4229922e3eb4e3eb6466"
+            ],
+            "layout": "IPY_MODEL_5d89a5574a3d4a8993e6dca78d406d2d"
+          }
+        },
+        "7751defbc4534d518d9e923b9019aa8b": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -1061,26 +936,10 @@
             "width": null
           }
         },
-        "db6a23e658a34722a8f22505c6ace7b4": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_name": "ProgressStyleModel",
-          "model_module_version": "1.5.0",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "ProgressStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "bar_color": null,
-            "description_width": ""
-          }
-        },
-        "7751defbc4534d518d9e923b9019aa8b": {
+        "84cc47dc70864bf3aa7599c06eb13c51": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -1129,25 +988,10 @@
             "width": null
           }
         },
-        "fe6352bce22a40e7a936e7f90313bd02": {
+        "9200f1303f124bddaa6114cdf0f5f878": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "68ff2fc00bd041e7b79a811e3de1e596": {
-          "model_module": "@jupyter-widgets/controls",
           "model_name": "HBoxModel",
-          "model_module_version": "1.5.0",
           "state": {
             "_dom_classes": [],
             "_model_module": "@jupyter-widgets/controls",
@@ -1159,62 +1003,17 @@
             "_view_name": "HBoxView",
             "box_style": "",
             "children": [
-              "IPY_MODEL_4c41e81bcd254df7b1265206a5a6b40b",
-              "IPY_MODEL_1a8c093fccbb437db6e0390a920f5cc5",
-              "IPY_MODEL_e11d04a9d22a4229922e3eb4e3eb6466"
+              "IPY_MODEL_17ddbb74e1764f37b8d34c311fae200c",
+              "IPY_MODEL_ef732739334b4ac593fd665e01cd83c1",
+              "IPY_MODEL_949ee3d1a9cd4060864dec5d4283ef2c"
             ],
-            "layout": "IPY_MODEL_5d89a5574a3d4a8993e6dca78d406d2d"
-          }
-        },
-        "4c41e81bcd254df7b1265206a5a6b40b": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_name": "HTMLModel",
-          "model_module_version": "1.5.0",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_dd24270dc07942a6972fbfaf58129989",
-            "placeholder": "​",
-            "style": "IPY_MODEL_643903cd7a5b4a52a4687ec38eb8c4dc",
-            "value": "Map: 100%"
+            "layout": "IPY_MODEL_b98629e053674527aacca899ab7f11a9"
           }
         },
-        "1a8c093fccbb437db6e0390a920f5cc5": {
+        "949ee3d1a9cd4060864dec5d4283ef2c": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "FloatProgressModel",
           "model_module_version": "1.5.0",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "FloatProgressModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "ProgressView",
-            "bar_style": "success",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_13ae11c314664c44ae18d35cf57a1334",
-            "max": 1044,
-            "min": 0,
-            "orientation": "horizontal",
-            "style": "IPY_MODEL_e68cfd05ba994a34b93107d2eab82ad3",
-            "value": 1044
-          }
-        },
-        "e11d04a9d22a4229922e3eb4e3eb6466": {
-          "model_module": "@jupyter-widgets/controls",
           "model_name": "HTMLModel",
-          "model_module_version": "1.5.0",
           "state": {
             "_dom_classes": [],
             "_model_module": "@jupyter-widgets/controls",
@@ -1226,68 +1025,16 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_ea283e7e8b234519b881c562b7eb01d3",
+            "layout": "IPY_MODEL_7751defbc4534d518d9e923b9019aa8b",
             "placeholder": "​",
-            "style": "IPY_MODEL_1ec5329ea0434df4b74d0f311e016c3e",
-            "value": " 1044/1044 [00:10&lt;00:00, 43.90 examples/s]"
+            "style": "IPY_MODEL_fe6352bce22a40e7a936e7f90313bd02",
+            "value": " 37/37 [00:00&lt;00:00, 3657.54it/s]"
           }
         },
-        "5d89a5574a3d4a8993e6dca78d406d2d": {
+        "b98629e053674527aacca899ab7f11a9": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "dd24270dc07942a6972fbfaf58129989": {
-          "model_module": "@jupyter-widgets/base",
           "model_name": "LayoutModel",
-          "model_module_version": "1.2.0",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -1336,25 +1083,26 @@
             "width": null
           }
         },
-        "643903cd7a5b4a52a4687ec38eb8c4dc": {
+        "db6a23e658a34722a8f22505c6ace7b4": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
           "state": {
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
+            "_model_name": "ProgressStyleModel",
             "_view_count": null,
             "_view_module": "@jupyter-widgets/base",
             "_view_module_version": "1.2.0",
             "_view_name": "StyleView",
+            "bar_color": null,
             "description_width": ""
           }
         },
-        "13ae11c314664c44ae18d35cf57a1334": {
+        "dd24270dc07942a6972fbfaf58129989": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -1403,10 +1151,31 @@
             "width": null
           }
         },
+        "e11d04a9d22a4229922e3eb4e3eb6466": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_ea283e7e8b234519b881c562b7eb01d3",
+            "placeholder": "​",
+            "style": "IPY_MODEL_1ec5329ea0434df4b74d0f311e016c3e",
+            "value": " 1044/1044 [00:10&lt;00:00, 43.90 examples/s]"
+          }
+        },
         "e68cfd05ba994a34b93107d2eab82ad3": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "ProgressStyleModel",
           "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
           "state": {
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
@@ -1421,8 +1190,8 @@
         },
         "ea283e7e8b234519b881c562b7eb01d3": {
           "model_module": "@jupyter-widgets/base",
-          "model_name": "LayoutModel",
           "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
           "state": {
             "_model_module": "@jupyter-widgets/base",
             "_model_module_version": "1.2.0",
@@ -1471,10 +1240,34 @@
             "width": null
           }
         },
-        "1ec5329ea0434df4b74d0f311e016c3e": {
+        "ef732739334b4ac593fd665e01cd83c1": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_3b80c66e0f384c45ab4187301599fab2",
+            "max": 37,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_db6a23e658a34722a8f22505c6ace7b4",
+            "value": 37
+          }
+        },
+        "fe6352bce22a40e7a936e7f90313bd02": {
           "model_module": "@jupyter-widgets/controls",
-          "model_name": "DescriptionStyleModel",
           "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
           "state": {
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
@@ -1491,4 +1284,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 5
-}
+}