"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Function to count words in a list of strings\n",
+ "def count_words(lst):\n",
+ " return [len(sent.split()) for sent in lst]\n",
+ "\n",
+ "# Count the number of words in questions and answers\n",
+ "question_word_counts = count_words(questions)\n",
+ "answer_word_counts = count_words(answers)\n",
+ "\n",
+ "# Create a DataFrame for better visualization\n",
+ "count_df = pd.DataFrame({\n",
+ " 'Question Word Count': question_word_counts,\n",
+ " 'Answer Word Count': answer_word_counts\n",
+ "})\n",
+ "\n",
+ "# Calculate mean and standard deviation\n",
+ "question_mean = np.mean(question_word_counts)\n",
+ "question_std = np.std(question_word_counts)\n",
+ "answer_mean = np.mean(answer_word_counts)\n",
+ "answer_std = np.std(answer_word_counts)\n",
+ "\n",
+ "# Plot the distribution of word counts\n",
+ "fig, axes = plt.subplots(1, 2, figsize=(11.5, 4.5))\n",
+ "\n",
+ "# Plot the distribution of question word counts\n",
+ "count_df['Question Word Count'].plot(kind='hist', ax=axes[0], bins=200, alpha=0.7, color='blue')\n",
+ "axes[0].set_title('Distribution of Question Word Counts')\n",
+ "axes[0].set_xlabel('Number of Words')\n",
+ "axes[0].set_ylabel('Frequency')\n",
+ "\n",
+ "# Set x-axis ticks to every 50th value\n",
+ "axes[0].xaxis.set_major_locator(mticker.MultipleLocator(50))\n",
+ "\n",
+ "# Rotate x-axis labels by 75 degrees\n",
+ "for label in axes[0].get_xticklabels():\n",
+ " label.set_rotation(75)\n",
+ "\n",
+ "# Add mean and sigma lines for questions\n",
+ "axes[0].axvline(question_mean, color='red', linestyle='dashed', linewidth=2, label='Mean')\n",
+ "axes[0].axvline(question_mean + question_std, color='orange', linestyle='dashed', linewidth=2, label='+1 Sigma')\n",
+ "axes[0].axvline(question_mean + 2 * question_std, color='gold', linestyle='dashed', linewidth=2, label='+2 Sigma')\n",
+ "axes[0].axvline(question_mean + 3 * question_std, color='yellow', linestyle='dashed', linewidth=2, label='+3 Sigma')\n",
+ "axes[0].axvline(question_mean + 4 * question_std, color='green', linestyle='dashed', linewidth=2, label='+4 Sigma')\n",
+ "axes[0].axvline(question_mean + 5 * question_std, color='blue', linestyle='dashed', linewidth=2, label='+5 Sigma')\n",
+ "axes[0].axvline(question_mean + 6 * question_std, color='purple', linestyle='dashed', linewidth=2, label='+6 Sigma')\n",
+ "axes[0].legend()\n",
+ "\n",
+ "# Plot the distribution of answer word counts\n",
+ "count_df['Answer Word Count'].plot(kind='hist', ax=axes[1], bins=200, alpha=0.7, color='green')\n",
+ "axes[1].set_title('Distribution of Answer Word Counts')\n",
+ "axes[1].set_xlabel('Number of Words')\n",
+ "axes[1].set_ylabel('Frequency')\n",
+ "\n",
+ "# Set x-axis ticks to every 50th value\n",
+ "axes[1].xaxis.set_major_locator(mticker.MultipleLocator(50))\n",
+ "\n",
+ "# Rotate x-axis labels by 75 degrees\n",
+ "for label in axes[1].get_xticklabels():\n",
+ " label.set_rotation(75)\n",
+ "\n",
+ "# Add mean and sigma lines for answers\n",
+ "axes[1].axvline(answer_mean, color='red', linestyle='dashed', linewidth=2, label='Mean')\n",
+ "axes[1].axvline(answer_mean + answer_std, color='orange', linestyle='dashed', linewidth=2, label='+1 Sigma')\n",
+ "axes[1].axvline(answer_mean + 2 * answer_std, color='gold', linestyle='dashed', linewidth=2, label='+2 Sigma')\n",
+ "axes[1].axvline(answer_mean + 3 * answer_std, color='yellow', linestyle='dashed', linewidth=2, label='+3 Sigma')\n",
+ "axes[1].axvline(answer_mean + 4 * answer_std, color='green', linestyle='dashed', linewidth=2, label='+4 Sigma')\n",
+ "axes[1].axvline(answer_mean + 5 * answer_std, color='blue', linestyle='dashed', linewidth=2, label='+5 Sigma')\n",
+ "axes[1].axvline(answer_mean + 6 * answer_std, color='purple', linestyle='dashed', linewidth=2, label='+6 Sigma')\n",
+ "axes[1].legend()\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2025-04-05T16:04:23.044645Z",
+ "iopub.status.busy": "2025-04-05T16:04:23.044418Z",
+ "iopub.status.idle": "2025-04-05T16:05:16.210701Z",
+ "shell.execute_reply": "2025-04-05T16:05:16.209889Z",
+ "shell.execute_reply.started": "2025-04-05T16:04:23.044626Z"
+ },
+ "id": "vo5WYfjuZvk0",
+ "trusted": true
+ },
+ "outputs": [],
+ "source": [
+ "# Tokenization questions and answers\n",
+ "questions = [word_tokenize(ques, format='text') for ques in questions]\n",
+ "answers = [word_tokenize(ans, format='text') for ans in answers]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2025-04-05T16:05:16.211850Z",
+ "iopub.status.busy": "2025-04-05T16:05:16.211601Z",
+ "iopub.status.idle": "2025-04-05T16:05:16.221098Z",
+ "shell.execute_reply": "2025-04-05T16:05:16.220295Z",
+ "shell.execute_reply.started": "2025-04-05T16:05:16.211830Z"
+ },
+ "id": "izz94RzyafH1",
+ "trusted": true
+ },
+ "outputs": [],
+ "source": [
+ "my_df = pd.DataFrame()\n",
+ "my_df['question'] = questions\n",
+ "my_df['answer'] = answers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2025-04-05T16:05:16.222191Z",
+ "iopub.status.busy": "2025-04-05T16:05:16.221897Z",
+ "iopub.status.idle": "2025-04-05T16:05:26.698706Z",
+ "shell.execute_reply": "2025-04-05T16:05:26.697517Z",
+ "shell.execute_reply.started": "2025-04-05T16:05:16.222163Z"
+ },
+ "id": "P2pHATnJZZeu",
+ "outputId": "804756ed-383e-4a70-8969-a72d295ef76a",
+ "trusted": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "a1ce8d22b60a4a51a840a851d5d1a843",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "tokenizer_config.json: 0%| | 0.00/82.0 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "608f6011768e4a64a374b71b44b4c4df",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "pytorch_model.bin: 0%| | 0.00/1.20G [00:00= self.patience:\n",
+ " control.should_training_stop = True\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "trusted": true
+ },
+ "outputs": [],
+ "source": [
+ "# Save model and tokenizer to \"vi-medical-transformer-fintune\"\n",
+ "os.makedirs(\"models\", exist_ok=True)\n",
+ "out_model_name = \"vi-medical-t5-finetune-qa\"\n",
+ "out_model_path = f\"models/{out_model_name}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2025-04-05T17:29:12.087285Z",
+ "iopub.status.busy": "2025-04-05T17:29:12.086983Z",
+ "iopub.status.idle": "2025-04-05T17:29:12.128730Z",
+ "shell.execute_reply": "2025-04-05T17:29:12.127895Z",
+ "shell.execute_reply.started": "2025-04-05T17:29:12.087263Z"
+ },
+ "id": "MvrLuxXssmCZ",
+ "outputId": "61cb24fa-b232-4afd-b752-2386a3717628",
+ "trusted": true
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
+ ]
+ }
+ ],
+ "source": [
+ "from transformers import TrainingArguments, Trainer\n",
+ "\n",
+ "training_args = TrainingArguments(\n",
+ " output_dir=out_model_path,\n",
+ " evaluation_strategy='epoch',\n",
+ " learning_rate=1e-4,\n",
+ " per_device_train_batch_size=2,\n",
+ " per_device_eval_batch_size=2,\n",
+ " num_train_epochs=26,\n",
+ " weight_decay=0.01,\n",
+ " save_total_limit=1,\n",
+ " load_best_model_at_end=True, # Load the best model at the end of training\n",
+ " metric_for_best_model='eval_loss', # Use evaluation loss as the metric\n",
+ " greater_is_better=False, # Lower evaluation loss is better\n",
+ " save_strategy='epoch' # Ensure save strategy matches evaluation strategy\n",
+ ")\n",
+ "\n",
+ "early_stopping_callback = EarlyStoppingCallback(patience=10)\n",
+ "\n",
+ "trainer = Trainer(\n",
+ " model=model,\n",
+ " args=training_args,\n",
+ " train_dataset=train_dataset,\n",
+ " eval_dataset=val_dataset,\n",
+ " callbacks=[early_stopping_callback]\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Training"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2025-04-05T17:29:14.852024Z",
+ "iopub.status.busy": "2025-04-05T17:29:14.851599Z",
+ "iopub.status.idle": "2025-04-06T01:44:27.669176Z",
+ "shell.execute_reply": "2025-04-06T01:44:27.668455Z",
+ "shell.execute_reply.started": "2025-04-05T17:29:14.851986Z"
+ },
+ "id": "7hm4r5GUZeaT",
+ "outputId": "6d638243-f051-4065-d04b-88bf55f95dd0",
+ "trusted": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " [85878/85878 8:15:11, Epoch 26/26]\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | Epoch | \n",
+ " Training Loss | \n",
+ " Validation Loss | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1 | \n",
+ " 0.473200 | \n",
+ " 0.395538 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0.406000 | \n",
+ " 0.382062 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0.398900 | \n",
+ " 0.377744 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 0.375000 | \n",
+ " 0.367524 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 0.405700 | \n",
+ " 0.360998 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 0.389800 | \n",
+ " 0.355099 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 0.393000 | \n",
+ " 0.349508 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 0.401700 | \n",
+ " 0.345235 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 0.360900 | \n",
+ " 0.343793 | \n",
+ "
\n",
+ " \n",
+ " | 10 | \n",
+ " 0.343600 | \n",
+ " 0.338723 | \n",
+ "
\n",
+ " \n",
+ " | 11 | \n",
+ " 0.363700 | \n",
+ " 0.336523 | \n",
+ "
\n",
+ " \n",
+ " | 12 | \n",
+ " 0.346400 | \n",
+ " 0.333973 | \n",
+ "
\n",
+ " \n",
+ " | 13 | \n",
+ " 0.327300 | \n",
+ " 0.332415 | \n",
+ "
\n",
+ " \n",
+ " | 14 | \n",
+ " 0.340600 | \n",
+ " 0.331806 | \n",
+ "
\n",
+ " \n",
+ " | 15 | \n",
+ " 0.344900 | \n",
+ " 0.330227 | \n",
+ "
\n",
+ " \n",
+ " | 16 | \n",
+ " 0.335900 | \n",
+ " 0.328180 | \n",
+ "
\n",
+ " \n",
+ " | 17 | \n",
+ " 0.332100 | \n",
+ " 0.326864 | \n",
+ "
\n",
+ " \n",
+ " | 18 | \n",
+ " 0.314700 | \n",
+ " 0.325996 | \n",
+ "
\n",
+ " \n",
+ " | 19 | \n",
+ " 0.325700 | \n",
+ " 0.326483 | \n",
+ "
\n",
+ " \n",
+ " | 20 | \n",
+ " 0.324400 | \n",
+ " 0.325702 | \n",
+ "
\n",
+ " \n",
+ " | 21 | \n",
+ " 0.302400 | \n",
+ " 0.323864 | \n",
+ "
\n",
+ " \n",
+ " | 22 | \n",
+ " 0.300700 | \n",
+ " 0.323292 | \n",
+ "
\n",
+ " \n",
+ " | 23 | \n",
+ " 0.307700 | \n",
+ " 0.322439 | \n",
+ "
\n",
+ " \n",
+ " | 24 | \n",
+ " 0.294700 | \n",
+ " 0.322978 | \n",
+ "
\n",
+ " \n",
+ " | 25 | \n",
+ " 0.299600 | \n",
+ " 0.322619 | \n",
+ "
\n",
+ " \n",
+ " | 26 | \n",
+ " 0.306100 | \n",
+ " 0.322764 | \n",
+ "
\n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "There were missing keys in the checkpoint model loaded: ['encoder.embed_tokens.weight', 'decoder.embed_tokens.weight'].\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "TrainOutput(global_step=85878, training_loss=0.3532296384634921, metrics={'train_runtime': 29708.0106, 'train_samples_per_second': 5.781, 'train_steps_per_second': 2.891, 'total_flos': 3.40508191555584e+16, 'train_loss': 0.3532296384634921, 'epoch': 26.0})"
+ ]
+ },
+ "execution_count": 28,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "trainer.train()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Model Evaluation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2025-04-06T01:47:59.247039Z",
+ "iopub.status.busy": "2025-04-06T01:47:59.246699Z",
+ "iopub.status.idle": "2025-04-06T01:48:34.369746Z",
+ "shell.execute_reply": "2025-04-06T01:48:34.368888Z",
+ "shell.execute_reply.started": "2025-04-06T01:47:59.247015Z"
+ },
+ "id": "wK3bwK-tZgB0",
+ "outputId": "c4840453-ec94-40b6-8171-d18caabe75f8",
+ "trusted": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " [367/367 00:35]\n",
+ "