Fine tuning bert-base to classify text.

Finetune BERT.ipynb

@@ -0,0 +1,512 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "73e72549-69f2-46b5-b0f5-655777139972",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:45:37.715126Z",
+     "iopub.status.busy": "2025-01-17T04:45:37.714808Z",
+     "iopub.status.idle": "2025-01-17T04:45:41.232154Z",
+     "shell.execute_reply": "2025-01-17T04:45:41.231851Z",
+     "shell.execute_reply.started": "2025-01-17T04:45:37.715090Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from datetime import datetime\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch import nn\n",
+    "from transformers import BertTokenizer, BertModel\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d4b79fb9-5e70-4600-8885-94bc0a6e917c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:45:41.232694Z",
+     "iopub.status.busy": "2025-01-17T04:45:41.232554Z",
+     "iopub.status.idle": "2025-01-17T04:45:41.236434Z",
+     "shell.execute_reply": "2025-01-17T04:45:41.236218Z",
+     "shell.execute_reply.started": "2025-01-17T04:45:41.232685Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def my_print(x):\n",
+    "    time_str = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
+    "    print(time_str, x)\n",
+    "\n",
+    "class BertClassifier(nn.Module):\n",
+    "    def __init__(self, num_classes: int = 8, bert_variety='bert-base-uncased'):\n",
+    "        super().__init__()\n",
+    "        self.bert = BertModel.from_pretrained(bert_variety)\n",
+    "        self.dropout = nn.Dropout(0.05)\n",
+    "        self.classifier = nn.Linear(self.bert.pooler.dense.out_features, num_classes)\n",
+    "\n",
+    "    def forward(self, input_ids, attention_mask):\n",
+    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
+    "        pooled_output = outputs.pooler_output\n",
+    "        pooled_output = self.dropout(pooled_output)\n",
+    "        logits = self.classifier(pooled_output)\n",
+    "        return logits\n",
+    "\n",
+    "class TextDataset(Dataset):\n",
+    "    def __init__(self, texts, labels, tokenizer, max_length=200):\n",
+    "        self.encodings = tokenizer(\n",
+    "            texts,\n",
+    "            truncation=True,\n",
+    "            padding=True,\n",
+    "            max_length=max_length,\n",
+    "            return_tensors='pt',\n",
+    "        )\n",
+    "        self.labels = torch.tensor([int(l[0]) for l in labels])\n",
+    "\n",
+    "    def __getitem__(self, idx):\n",
+    "        item = {key: val[idx] for key, val in self.encodings.items()}\n",
+    "        item['labels'] = self.labels[idx]\n",
+    "        return item\n",
+    "\n",
+    "    def __len__(self) -> int:\n",
+    "        return len(self.labels)\n",
+    "\n",
+    "def train_model(model, train_dataloader, device, num_epochs):\n",
+    "    optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5)\n",
+    "    criterion = nn.CrossEntropyLoss()\n",
+    "    model.train()\n",
+    "\n",
+    "    my_print('Starting epoch 1.')\n",
+    "    for epoch in range(num_epochs):\n",
+    "        total_loss = 0\n",
+    "        for batch in train_dataloader:\n",
+    "            optimizer.zero_grad()\n",
+    "\n",
+    "            input_ids = batch['input_ids'].to(device)\n",
+    "            attention_mask = batch['attention_mask'].to(device)\n",
+    "            labels = batch['labels'].to(device)\n",
+    "\n",
+    "            outputs = model(input_ids, attention_mask)\n",
+    "            loss = criterion(outputs, labels)\n",
+    "\n",
+    "            loss.backward()\n",
+    "            optimizer.step()\n",
+    "\n",
+    "            total_loss += loss.item()\n",
+    "        avg_loss = total_loss / len(train_dataloader)\n",
+    "        my_print(f'Epoch {epoch+1}/{num_epochs} done, Average Loss: {avg_loss:0.4f}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "07131bce-23ad-4787-8622-cce401f3e5ce",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:45:41.237451Z",
+     "iopub.status.busy": "2025-01-17T04:45:41.237358Z",
+     "iopub.status.idle": "2025-01-17T04:45:41.252075Z",
+     "shell.execute_reply": "2025-01-17T04:45:41.251851Z",
+     "shell.execute_reply.started": "2025-01-17T04:45:41.237443Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "if torch.backends.mps.is_available():\n",
+    "    device = torch.device('mps')\n",
+    "    torch.mps.empty_cache()\n",
+    "elif torch.cuda.is_available():\n",
+    "    device = torch.device('cuda')\n",
+    "else:\n",
+    "    device = torch.device('cpu')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "695bc080-bbd7-4937-af5b-50db1c936500",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:45:41.252581Z",
+     "iopub.status.busy": "2025-01-17T04:45:41.252476Z",
+     "iopub.status.idle": "2025-01-17T04:45:41.255279Z",
+     "shell.execute_reply": "2025-01-17T04:45:41.255045Z",
+     "shell.execute_reply.started": "2025-01-17T04:45:41.252572Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def run_training(\n",
+    "    max_dataset_size=16 * 200,\n",
+    "    bert_variety='bert-base-uncased',\n",
+    "    max_length=200,\n",
+    "    num_epochs=3,\n",
+    "    batch_size=32,\n",
+    "):\n",
+    "    hf_dataset = load_dataset(\"quotaclimat/frugalaichallenge-text-train\")\n",
+    "    if not max_dataset_size == 'full' and max_dataset_size < len(hf_dataset['train']):\n",
+    "        train_dataset = hf_dataset['train'][:max_dataset_size]\n",
+    "    else:\n",
+    "        train_dataset = hf_dataset['train']\n",
+    "    \n",
+    "    tokenizer = BertTokenizer.from_pretrained(bert_variety, max_length=max_length)\n",
+    "    model = BertClassifier(bert_variety=bert_variety)\n",
+    "    if torch.backends.mps.is_available():\n",
+    "        device = torch.device('mps')\n",
+    "        torch.mps.empty_cache()\n",
+    "    elif torch.cuda.is_available():\n",
+    "        device = torch.device('cuda')\n",
+    "    else:\n",
+    "        device = torch.device('cpu')\n",
+    "    model.to(device)\n",
+    "    \n",
+    "    dataset = TextDataset(\n",
+    "        train_dataset['quote'],\n",
+    "        train_dataset['label'],\n",
+    "        tokenizer=tokenizer,\n",
+    "        max_length=max_length,\n",
+    "    )\n",
+    "    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n",
+    "    \n",
+    "    train_model(model, dataloader, device, num_epochs=num_epochs)\n",
+    "    return model, tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "792fd13f-e7cc-4d90-832d-c0da15e193cd",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:45:41.255750Z",
+     "iopub.status.busy": "2025-01-17T04:45:41.255661Z",
+     "iopub.status.idle": "2025-01-17T04:47:17.151654Z",
+     "shell.execute_reply": "2025-01-17T04:47:17.149076Z",
+     "shell.execute_reply.started": "2025-01-17T04:45:41.255742Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 20:45:45 Starting epoch 1.\n",
+      "2025-01-16 20:46:15 Epoch 1/3 done, Average Loss: 1.9223\n",
+      "2025-01-16 20:46:46 Epoch 2/3 done, Average Loss: 1.6052\n",
+      "2025-01-16 20:47:17 Epoch 3/3 done, Average Loss: 1.2876\n"
+     ]
+    }
+   ],
+   "source": [
+    "model, tokenizer = run_training(\n",
+    "    max_dataset_size=16 * 50,\n",
+    "    bert_variety='bert-base-uncased',\n",
+    "    max_length=200,\n",
+    "    num_epochs=3,\n",
+    "    batch_size=32,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0aedfcca-843e-4f4c-8062-3e4625161bcc",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:47:17.158101Z",
+     "iopub.status.busy": "2025-01-17T04:47:17.157305Z",
+     "iopub.status.idle": "2025-01-17T04:47:17.333568Z",
+     "shell.execute_reply": "2025-01-17T04:47:17.333317Z",
+     "shell.execute_reply.started": "2025-01-17T04:47:17.157437Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 20:47:17 Predictions: tensor([6, 1, 1, 6, 1, 6, 6], device='mps:0')\n"
+     ]
+    }
+   ],
+   "source": [
+    "model.eval()\n",
+    "test_text = [\n",
+    "    'This was a great experience!', # 0_not_relevant\n",
+    "    'My favorite hike is Laguna de los Tres.', # 0_not_relevant\n",
+    "    'Crops will grow great in Finland if it\\'s warmer there.', # 3_not_bad\n",
+    "    'Climate change is fake.', # 1_not_happening\n",
+    "    'The apparent warming is caused by solar cycles.', # 2_not_human\n",
+    "    'Solar panels emit bad vibes.', # 4_solutions_harmful_unnecessary\n",
+    "    'All those so-called scientists are Democrats.', # 6_proponents_biased\n",
+    "]\n",
+    "test_encoding = tokenizer(\n",
+    "    test_text,\n",
+    "    truncation=True,\n",
+    "    padding=True,\n",
+    "    return_tensors='pt',\n",
+    ")\n",
+    "\n",
+    "with torch.no_grad():\n",
+    "    test_input_ids = test_encoding['input_ids'].to(device)\n",
+    "    test_attention_mask = test_encoding['attention_mask'].to(device)\n",
+    "    outputs = model(test_input_ids, test_attention_mask)\n",
+    "    predictions = torch.argmax(outputs, dim=1)\n",
+    "    my_print(f'Predictions: {predictions}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "881b738e-2392-4b7e-a0de-a0bad572ddfa",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:47:17.334399Z",
+     "iopub.status.busy": "2025-01-17T04:47:17.334287Z",
+     "iopub.status.idle": "2025-01-17T04:50:59.116389Z",
+     "shell.execute_reply": "2025-01-17T04:50:59.115528Z",
+     "shell.execute_reply.started": "2025-01-17T04:47:17.334390Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 20:47:23 Starting epoch 1.\n",
+      "2025-01-16 20:48:35 Epoch 1/3 done, Average Loss: 1.4272\n",
+      "2025-01-16 20:49:46 Epoch 2/3 done, Average Loss: 0.8694\n",
+      "2025-01-16 20:50:59 Epoch 3/3 done, Average Loss: 0.5774\n"
+     ]
+    }
+   ],
+   "source": [
+    "model, tokenizer = run_training(\n",
+    "    max_dataset_size='full',\n",
+    "    bert_variety='bert-base-uncased',\n",
+    "    max_length=64,\n",
+    "    num_epochs=3,\n",
+    "    batch_size=32,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "1d29336e-7f88-4127-afdf-2fe043e310e1",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:50:59.118025Z",
+     "iopub.status.busy": "2025-01-17T04:50:59.117838Z",
+     "iopub.status.idle": "2025-01-17T04:58:02.423121Z",
+     "shell.execute_reply": "2025-01-17T04:58:02.421532Z",
+     "shell.execute_reply.started": "2025-01-17T04:50:59.118005Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 20:51:04 Starting epoch 1.\n",
+      "2025-01-16 20:53:20 Epoch 1/3 done, Average Loss: 1.4107\n",
+      "2025-01-16 20:55:41 Epoch 2/3 done, Average Loss: 0.8491\n",
+      "2025-01-16 20:58:02 Epoch 3/3 done, Average Loss: 0.5359\n"
+     ]
+    }
+   ],
+   "source": [
+    "model, tokenizer = run_training(\n",
+    "    max_dataset_size='full',\n",
+    "    bert_variety='bert-base-uncased',\n",
+    "    max_length=128,\n",
+    "    num_epochs=3,\n",
+    "    batch_size=32,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "461b8f57-0c52-403a-bb69-3bc192b323bf",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T04:58:02.426159Z",
+     "iopub.status.busy": "2025-01-17T04:58:02.425896Z",
+     "iopub.status.idle": "2025-01-17T05:05:36.903446Z",
+     "shell.execute_reply": "2025-01-17T05:05:36.901961Z",
+     "shell.execute_reply.started": "2025-01-17T04:58:02.426132Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 20:58:08 Starting epoch 1.\n",
+      "2025-01-16 21:00:38 Epoch 1/3 done, Average Loss: 1.2946\n",
+      "2025-01-16 21:03:07 Epoch 2/3 done, Average Loss: 0.7425\n",
+      "2025-01-16 21:05:36 Epoch 3/3 done, Average Loss: 0.4126\n"
+     ]
+    }
+   ],
+   "source": [
+    "model, tokenizer = run_training(\n",
+    "    max_dataset_size='full',\n",
+    "    bert_variety='bert-base-uncased',\n",
+    "    max_length=128,\n",
+    "    num_epochs=3,\n",
+    "    batch_size=16,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "28354e8c-886a-4523-8968-8c688c13f6a3",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T05:05:36.905668Z",
+     "iopub.status.busy": "2025-01-17T05:05:36.905353Z",
+     "iopub.status.idle": "2025-01-17T05:21:10.045463Z",
+     "shell.execute_reply": "2025-01-17T05:21:10.044788Z",
+     "shell.execute_reply.started": "2025-01-17T05:05:36.905630Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 21:05:43 Starting epoch 1.\n",
+      "2025-01-16 21:10:53 Epoch 1/3 done, Average Loss: 1.3415\n",
+      "2025-01-16 21:16:02 Epoch 2/3 done, Average Loss: 0.7216\n",
+      "2025-01-16 21:21:10 Epoch 3/3 done, Average Loss: 0.3978\n"
+     ]
+    }
+   ],
+   "source": [
+    "model, tokenizer = run_training(\n",
+    "    max_dataset_size='full',\n",
+    "    bert_variety='bert-base-uncased',\n",
+    "    max_length=256,\n",
+    "    num_epochs=3,\n",
+    "    batch_size=16,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "e3b099c6-6b98-473b-8797-5032213b9fcb",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T05:21:10.059844Z",
+     "iopub.status.busy": "2025-01-17T05:21:10.058980Z",
+     "iopub.status.idle": "2025-01-17T05:21:10.164116Z",
+     "shell.execute_reply": "2025-01-17T05:21:10.163826Z",
+     "shell.execute_reply.started": "2025-01-17T05:21:10.059552Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-01-16 21:21:10 Predictions: tensor([0, 0, 3, 6, 2, 4, 6], device='mps:0')\n"
+     ]
+    }
+   ],
+   "source": [
+    "model.eval()\n",
+    "test_text = [\n",
+    "    'This was a great experience!', # 0_not_relevant\n",
+    "    'My favorite hike is Laguna de los Tres.', # 0_not_relevant\n",
+    "    'Crops will grow great in Finland if it\\'s warmer there.', # 3_not_bad\n",
+    "    'Climate change is fake.', # 1_not_happening\n",
+    "    'The apparent warming is caused by solar cycles.', # 2_not_human\n",
+    "    'Solar panels emit bad vibes.', # 4_solutions_harmful_unnecessary\n",
+    "    'All those so-called scientists are Democrats.', # 6_proponents_biased\n",
+    "]\n",
+    "test_encoding = tokenizer(\n",
+    "    test_text,\n",
+    "    truncation=True,\n",
+    "    padding=True,\n",
+    "    return_tensors='pt',\n",
+    ")\n",
+    "\n",
+    "with torch.no_grad():\n",
+    "    test_input_ids = test_encoding['input_ids'].to(device)\n",
+    "    test_attention_mask = test_encoding['attention_mask'].to(device)\n",
+    "    outputs = model(test_input_ids, test_attention_mask)\n",
+    "    predictions = torch.argmax(outputs, dim=1)\n",
+    "    my_print(f'Predictions: {predictions}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "befb94b5-88bf-40fc-8b26-cf373d1256e0",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-01-17T05:27:58.042752Z",
+     "iopub.status.busy": "2025-01-17T05:27:58.042151Z",
+     "iopub.status.idle": "2025-01-17T05:27:58.454054Z",
+     "shell.execute_reply": "2025-01-17T05:27:58.453644Z",
+     "shell.execute_reply.started": "2025-01-17T05:27:58.042662Z"
+    }
+   },
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "'BertClassifier' object has no attribute 'push_to_hub'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[12], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpush_to_hub\u001b[49m()\n",
+      "File \u001b[0;32m~/miniconda3/envs/py313/lib/python3.13/site-packages/torch/nn/modules/module.py:1931\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1929\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m   1930\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1931\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[1;32m   1932\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1933\u001b[0m )\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'BertClassifier' object has no attribute 'push_to_hub'"
+     ]
+    }
+   ],
+   "source": [
+    "model.push_to_hub()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "251ef9ee-8ba3-495f-8fe6-a93aa63168ce",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

tasks/text.py

@@ -12,7 +12,7 @@ router = APIRouter()
 DESCRIPTION = "Most common class baseline"
 ROUTE = "/text"
 
-def baseline_model(dataset_length):
+def baseline_model(dataset_length: int):
     # Make random predictions (placeholder for actual model inference)
     #predictions = [random.randint(0, 7) for _ in range(dataset_length)]