{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cnn.py",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMqcNjDUZSdpo72P+kCl6Fo",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/emiliomrb/AprendizajeProfundo/blob/master/cnn_py.py\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14I00h-gb453"
      },
      "source": [
        "import argparse\n",
        "import gzip\n",
        "import json\n",
        "import logging\n",
        "import mlflow\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from sklearn.metrics import balanced_accuracy_score\n",
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from .dataset import MeliChallengeDataset\n",
        "from .utils import PadSequences\n",
        "\n",
        "\n",
        "logging.basicConfig(\n",
        "    format=\"%(asctime)s: %(levelname)s - %(message)s\",\n",
        "    level=logging.INFO\n",
        ")\n",
        "\n",
        "\n",
        "class CNNClassifier(nn.Module):\n",
        "    def __init__(self,\n",
        "                 pretrained_embeddings_path,\n",
        "                 token_to_index,\n",
        "                 n_labels,\n",
        "                 dropout=0.3,\n",
        "                 vector_size=300,\n",
        "                 filters_count = 100,\n",
        "                 filters_lenght= [2,3,4],\n",
        "                 freeze_embedings=True):\n",
        "        super().__init__()\n",
        "        with gzip.open(token_to_index, \"rt\") as fh:\n",
        "            token_to_index = json.load(fh)\n",
        "        embeddings_matrix = torch.randn(len(token_to_index), vector_size)\n",
        "        embeddings_matrix[0] = torch.zeros(vector_size)\n",
        "        with gzip.open(pretrained_embeddings_path, \"rt\") as fh:\n",
        "            next(fh)\n",
        "            for line in fh:\n",
        "                word, vector = line.strip().split(None, 1)\n",
        "                if word in token_to_index:\n",
        "                    embeddings_matrix[token_to_index[word]] =\\\n",
        "                        torch.FloatTensor([float(n) for n in vector.split()])\n",
        "        self.embeddings = nn.Embedding.from_pretrained(embeddings_matrix,\n",
        "                                                       freeze=freeze_embedings,\n",
        "                                                       padding_idx=0)\n",
        "      \n",
        "\n",
        "        self.convs = []\n",
        "        for filter in self.filters_lenght:\n",
        "            self.convs.append(\n",
        "                nn.Conv1d(vector_size, self.filters_count, filter)\n",
        "            )\n",
        "        self.convs = nn.ModuleList(self.convs)\n",
        "        self.fc = nn.Linear(filters_count * len(filters_lenght), 128)\n",
        "        self.output = nn.Linear(128, n_labels)\n",
        "        self.vector_size = vector_size\n",
        "    \n",
        "    @staticmethod\n",
        "    def conv_global_max_pool(x, conv):\n",
        "        return F.relu(conv(x).transpose(1, 2).max(1)[0])\n",
        "    \n",
        "    def forward(self, x):\n",
        "        x = self.embeddings(x).transpose(1, 2)  # Conv1d takes (batch, channel, seq_len)\n",
        "        x = [self.conv_global_max_pool(x, conv) for conv in self.convs]\n",
        "        x = torch.cat(x, dim=1)\n",
        "        x = F.relu(self.fc(x))\n",
        "        x = torch.softmax(self.output(x))\n",
        "        return x\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    parser = argparse.ArgumentParser()\n",
        "    parser.add_argument(\"--train-data\",\n",
        "                        help=\"Path to the the training dataset\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--token-to-index\",\n",
        "                        help=\"Path to the the json file that maps tokens to indices\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--pretrained-embeddings\",\n",
        "                        help=\"Path to the pretrained embeddings file.\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--language\",\n",
        "                        help=\"Language working with\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--test-data\",\n",
        "                        help=\"If given, use the test data to perform evaluation.\")\n",
        "    parser.add_argument(\"--validation-data\",\n",
        "                        help=\"If given, use the validation data to perform evaluation.\")\n",
        "    parser.add_argument(\"--embeddings-size\",\n",
        "                        default=300,\n",
        "                        help=\"Size of the vectors.\",\n",
        "                        type=int)\n",
        "    parser.add_argument(\"--hidden-layers\",\n",
        "                        help=\"Sizes of the hidden layers of the MLP (can be one or more values)\",\n",
        "                        nargs=\"+\",\n",
        "                        default=[256, 128],\n",
        "                        type=int)\n",
        "    parser.add_argument(\"--dropout\",\n",
        "                        help=\"Dropout to apply to each hidden layer\",\n",
        "                        default=0.3,\n",
        "                        type=float)\n",
        "    parser.add_argument(\"--epochs\",\n",
        "                        help=\"Number of epochs\",\n",
        "                        default=3,\n",
        "                        type=int)\n",
        "\n",
        "    args = parser.parse_args()\n",
        "\n",
        "    pad_sequences = PadSequences(\n",
        "        pad_value=0,\n",
        "        max_length=None,\n",
        "        min_length=max(filters_lenght) #max conv\n",
        "    )\n",
        "\n",
        "    logging.info(\"Building training dataset\")\n",
        "    train_dataset = MeliChallengeDataset(\n",
        "        dataset_path=args.train_data,\n",
        "        random_buffer_size=2048  # This can be a hypterparameter\n",
        "    )\n",
        "    train_loader = DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=128,  # This can be a hyperparameter\n",
        "        shuffle=False,\n",
        "        collate_fn=pad_sequences,\n",
        "        drop_last=False\n",
        "    )\n",
        "\n",
        "    if args.validation_data:\n",
        "        logging.info(\"Building validation dataset\")\n",
        "        validation_dataset = MeliChallengeDataset(\n",
        "            dataset_path=args.validation_data,\n",
        "            random_buffer_size=1\n",
        "        )\n",
        "        validation_loader = DataLoader(\n",
        "            validation_dataset,\n",
        "            batch_size=128,\n",
        "            shuffle=False,\n",
        "            collate_fn=pad_sequences,\n",
        "            drop_last=False\n",
        "        )\n",
        "    else:\n",
        "        validation_dataset = None\n",
        "        validation_loader = None\n",
        "\n",
        "    if args.test_data:\n",
        "        logging.info(\"Building test dataset\")\n",
        "        test_dataset = MeliChallengeDataset(\n",
        "            dataset_path=args.test_data,\n",
        "            random_buffer_size=1\n",
        "        )\n",
        "        test_loader = DataLoader(\n",
        "            test_dataset,\n",
        "            batch_size=128,\n",
        "            shuffle=False,\n",
        "            collate_fn=pad_sequences,\n",
        "            drop_last=False\n",
        "        )\n",
        "    else:\n",
        "        test_dataset = None\n",
        "        test_loader = None\n",
        "\n",
        "    mlflow.set_experiment(f\"diplodatos.{args.language}\")\n",
        "\n",
        "    with mlflow.start_run():\n",
        "        logging.info(\"Starting experiment\")\n",
        "        # Log all relevent hyperparameters\n",
        "        mlflow.log_params({\n",
        "            \"model_type\": \"Multilayer Perceptron\",\n",
        "            \"embeddings\": args.pretrained_embeddings,\n",
        "            \"hidden_layers\": args.hidden_layers,\n",
        "            \"dropout\": args.dropout,\n",
        "            \"embeddings_size\": args.embeddings_size,\n",
        "            \"epochs\": args.epochs\n",
        "        })\n",
        "        device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
        "\n",
        "        logging.info(\"Building classifier\")\n",
        "        model = MLPClassifier(\n",
        "            pretrained_embeddings_path=args.pretrained_embeddings,\n",
        "            token_to_index=args.token_to_index,\n",
        "            n_labels=train_dataset.n_labels,\n",
        "            hidden_layers=args.hidden_layers,\n",
        "            dropout=args.dropout,\n",
        "            vector_size=args.embeddings_size,\n",
        "            freeze_embedings=True  # This can be a hyperparameter\n",
        "        )\n",
        "        model = model.to(device)\n",
        "        loss = nn.CrossEntropyLoss()\n",
        "        optimizer = optim.Adam(\n",
        "            model.parameters(),\n",
        "            lr=1e-3,  # This can be a hyperparameter\n",
        "            weight_decay=1e-5  # This can be a hyperparameter\n",
        "        )\n",
        "\n",
        "        logging.info(\"Training classifier\")\n",
        "        for epoch in trange(args.epochs):\n",
        "            model.train()\n",
        "            running_loss = []\n",
        "            for idx, batch in enumerate(tqdm(train_loader)):\n",
        "                optimizer.zero_grad()\n",
        "                data = batch[\"data\"].to(device)\n",
        "                target = batch[\"target\"].to(device)\n",
        "                output = model(data)\n",
        "                loss_value = loss(output, target)\n",
        "                loss_value.backward()\n",
        "                optimizer.step()\n",
        "                running_loss.append(loss_value.item())\n",
        "            mlflow.log_metric(\"train_loss\", sum(running_loss) / len(running_loss), epoch)\n",
        "\n",
        "            if validation_dataset:\n",
        "                logging.info(\"Evaluating model on validation\")\n",
        "                model.eval()\n",
        "                running_loss = []\n",
        "                targets = []\n",
        "                predictions = []\n",
        "                with torch.no_grad():\n",
        "                    for batch in tqdm(validation_loader):\n",
        "                        data = batch[\"data\"].to(device)\n",
        "                        target = batch[\"target\"].to(device)\n",
        "                        output = model(data)\n",
        "                        running_loss.append(\n",
        "                            loss(output, target).item()\n",
        "                        )\n",
        "                        targets.extend(batch[\"target\"].numpy())\n",
        "                        predictions.extend(output.argmax(axis=1).detach().cpu().numpy())\n",
        "                    mlflow.log_metric(\"validation_loss\", sum(running_loss) / len(running_loss), epoch)\n",
        "                    mlflow.log_metric(\"validation_bacc\", balanced_accuracy_score(targets, predictions), epoch)\n",
        "\n",
        "        if test_dataset:\n",
        "            logging.info(\"Evaluating model on test\")\n",
        "            model.eval()\n",
        "            running_loss = []\n",
        "            targets = []\n",
        "            predictions = []\n",
        "            with torch.no_grad():\n",
        "                for batch in tqdm(test_loader):\n",
        "                    data = batch[\"data\"].to(device)\n",
        "                    target = batch[\"target\"].to(device)\n",
        "                    output = model(data)\n",
        "                    running_loss.append(\n",
        "                        loss(output, target).item()\n",
        "                    )\n",
        "                    targets.extend(batch[\"target\"].numpy())\n",
        "                    predictions.extend(output.argmax(axis=1).detach().cpu().numpy())\n",
        "                mlflow.log_metric(\"test_loss\", sum(running_loss) / len(running_loss), epoch)\n",
        "                mlflow.log_metric(\"test_bacc\", balanced_accuracy_score(targets, predictions), epoch)\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}
{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cnn.py",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMqcNjDUZSdpo72P+kCl6Fo",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/emiliomrb/AprendizajeProfundo/blob/master/cnn_py.py\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14I00h-gb453"
      },
      "source": [
        "import argparse\n",
        "import gzip\n",
        "import json\n",
        "import logging\n",
        "import mlflow\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from sklearn.metrics import balanced_accuracy_score\n",
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from .dataset import MeliChallengeDataset\n",
        "from .utils import PadSequences\n",
        "\n",
        "\n",
        "logging.basicConfig(\n",
        "    format=\"%(asctime)s: %(levelname)s - %(message)s\",\n",
        "    level=logging.INFO\n",
        ")\n",
        "\n",
        "\n",
        "class CNNClassifier(nn.Module):\n",
        "    def __init__(self,\n",
        "                 pretrained_embeddings_path,\n",
        "                 token_to_index,\n",
        "                 n_labels,\n",
        "                 dropout=0.3,\n",
        "                 vector_size=300,\n",
        "                 filters_count = 100,\n",
        "                 filters_lenght= [2,3,4],\n",
        "                 freeze_embedings=True):\n",
        "        super().__init__()\n",
        "        with gzip.open(token_to_index, \"rt\") as fh:\n",
        "            token_to_index = json.load(fh)\n",
        "        embeddings_matrix = torch.randn(len(token_to_index), vector_size)\n",
        "        embeddings_matrix[0] = torch.zeros(vector_size)\n",
        "        with gzip.open(pretrained_embeddings_path, \"rt\") as fh:\n",
        "            next(fh)\n",
        "            for line in fh:\n",
        "                word, vector = line.strip().split(None, 1)\n",
        "                if word in token_to_index:\n",
        "                    embeddings_matrix[token_to_index[word]] =\\\n",
        "                        torch.FloatTensor([float(n) for n in vector.split()])\n",
        "        self.embeddings = nn.Embedding.from_pretrained(embeddings_matrix,\n",
        "                                                       freeze=freeze_embedings,\n",
        "                                                       padding_idx=0)\n",
        "      \n",
        "\n",
        "        self.convs = []\n",
        "        for filter in self.filters_lenght:\n",
        "            self.convs.append(\n",
        "                nn.Conv1d(vector_size, self.filters_count, filter)\n",
        "            )\n",
        "        self.convs = nn.ModuleList(self.convs)\n",
        "        self.fc = nn.Linear(filters_count * len(filters_lenght), 128)\n",
        "        self.output = nn.Linear(128, n_labels)\n",
        "        self.vector_size = vector_size\n",
        "    \n",
        "    @staticmethod\n",
        "    def conv_global_max_pool(x, conv):\n",
        "        return F.relu(conv(x).transpose(1, 2).max(1)[0])\n",
        "    \n",
        "    def forward(self, x):\n",
        "        x = self.embeddings(x).transpose(1, 2)  # Conv1d takes (batch, channel, seq_len)\n",
        "        x = [self.conv_global_max_pool(x, conv) for conv in self.convs]\n",
        "        x = torch.cat(x, dim=1)\n",
        "        x = F.relu(self.fc(x))\n",
        "        x = torch.softmax(self.output(x))\n",
        "        return x\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    parser = argparse.ArgumentParser()\n",
        "    parser.add_argument(\"--train-data\",\n",
        "                        help=\"Path to the the training dataset\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--token-to-index\",\n",
        "                        help=\"Path to the the json file that maps tokens to indices\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--pretrained-embeddings\",\n",
        "                        help=\"Path to the pretrained embeddings file.\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--language\",\n",
        "                        help=\"Language working with\",\n",
        "                        required=True)\n",
        "    parser.add_argument(\"--test-data\",\n",
        "                        help=\"If given, use the test data to perform evaluation.\")\n",
        "    parser.add_argument(\"--validation-data\",\n",
        "                        help=\"If given, use the validation data to perform evaluation.\")\n",
        "    parser.add_argument(\"--embeddings-size\",\n",
        "                        default=300,\n",
        "                        help=\"Size of the vectors.\",\n",
        "                        type=int)\n",
        "    parser.add_argument(\"--hidden-layers\",\n",
        "                        help=\"Sizes of the hidden layers of the MLP (can be one or more values)\",\n",
        "                        nargs=\"+\",\n",
        "                        default=[256, 128],\n",
        "                        type=int)\n",
        "    parser.add_argument(\"--dropout\",\n",
        "                        help=\"Dropout to apply to each hidden layer\",\n",
        "                        default=0.3,\n",
        "                        type=float)\n",
        "    parser.add_argument(\"--epochs\",\n",
        "                        help=\"Number of epochs\",\n",
        "                        default=3,\n",
        "                        type=int)\n",
        "\n",
        "    args = parser.parse_args()\n",
        "\n",
        "    pad_sequences = PadSequences(\n",
        "        pad_value=0,\n",
        "        max_length=None,\n",
        "        min_length=max(filters_lenght) #max conv\n",
        "    )\n",
        "\n",
        "    logging.info(\"Building training dataset\")\n",
        "    train_dataset = MeliChallengeDataset(\n",
        "        dataset_path=args.train_data,\n",
        "        random_buffer_size=2048  # This can be a hypterparameter\n",
        "    )\n",
        "    train_loader = DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=128,  # This can be a hyperparameter\n",
        "        shuffle=False,\n",
        "        collate_fn=pad_sequences,\n",
        "        drop_last=False\n",
        "    )\n",
        "\n",
        "    if args.validation_data:\n",
        "        logging.info(\"Building validation dataset\")\n",
        "        validation_dataset = MeliChallengeDataset(\n",
        "            dataset_path=args.validation_data,\n",
        "            random_buffer_size=1\n",
        "        )\n",
        "        validation_loader = DataLoader(\n",
        "            validation_dataset,\n",
        "            batch_size=128,\n",
        "            shuffle=False,\n",
        "            collate_fn=pad_sequences,\n",
        "            drop_last=False\n",
        "        )\n",
        "    else:\n",
        "        validation_dataset = None\n",
        "        validation_loader = None\n",
        "\n",
        "    if args.test_data:\n",
        "        logging.info(\"Building test dataset\")\n",
        "        test_dataset = MeliChallengeDataset(\n",
        "            dataset_path=args.test_data,\n",
        "            random_buffer_size=1\n",
        "        )\n",
        "        test_loader = DataLoader(\n",
        "            test_dataset,\n",
        "            batch_size=128,\n",
        "            shuffle=False,\n",
        "            collate_fn=pad_sequences,\n",
        "            drop_last=False\n",
        "        )\n",
        "    else:\n",
        "        test_dataset = None\n",
        "        test_loader = None\n",
        "\n",
        "    mlflow.set_experiment(f\"diplodatos.{args.language}\")\n",
        "\n",
        "    with mlflow.start_run():\n",
        "        logging.info(\"Starting experiment\")\n",
        "        # Log all relevent hyperparameters\n",
        "        mlflow.log_params({\n",
        "            \"model_type\": \"Multilayer Perceptron\",\n",
        "            \"embeddings\": args.pretrained_embeddings,\n",
        "            \"hidden_layers\": args.hidden_layers,\n",
        "            \"dropout\": args.dropout,\n",
        "            \"embeddings_size\": args.embeddings_size,\n",
        "            \"epochs\": args.epochs\n",
        "        })\n",
        "        device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
        "\n",
        "        logging.info(\"Building classifier\")\n",
        "        model = MLPClassifier(\n",
        "            pretrained_embeddings_path=args.pretrained_embeddings,\n",
        "            token_to_index=args.token_to_index,\n",
        "            n_labels=train_dataset.n_labels,\n",
        "            hidden_layers=args.hidden_layers,\n",
        "            dropout=args.dropout,\n",
        "            vector_size=args.embeddings_size,\n",
        "            freeze_embedings=True  # This can be a hyperparameter\n",
        "        )\n",
        "        model = model.to(device)\n",
        "        loss = nn.CrossEntropyLoss()\n",
        "        optimizer = optim.Adam(\n",
        "            model.parameters(),\n",
        "            lr=1e-3,  # This can be a hyperparameter\n",
        "            weight_decay=1e-5  # This can be a hyperparameter\n",
        "        )\n",
        "\n",
        "        logging.info(\"Training classifier\")\n",
        "        for epoch in trange(args.epochs):\n",
        "            model.train()\n",
        "            running_loss = []\n",
        "            for idx, batch in enumerate(tqdm(train_loader)):\n",
        "                optimizer.zero_grad()\n",
        "                data = batch[\"data\"].to(device)\n",
        "                target = batch[\"target\"].to(device)\n",
        "                output = model(data)\n",
        "                loss_value = loss(output, target)\n",
        "                loss_value.backward()\n",
        "                optimizer.step()\n",
        "                running_loss.append(loss_value.item())\n",
        "            mlflow.log_metric(\"train_loss\", sum(running_loss) / len(running_loss), epoch)\n",
        "\n",
        "            if validation_dataset:\n",
        "                logging.info(\"Evaluating model on validation\")\n",
        "                model.eval()\n",
        "                running_loss = []\n",
        "                targets = []\n",
        "                predictions = []\n",
        "                with torch.no_grad():\n",
        "                    for batch in tqdm(validation_loader):\n",
        "                        data = batch[\"data\"].to(device)\n",
        "                        target = batch[\"target\"].to(device)\n",
        "                        output = model(data)\n",
        "                        running_loss.append(\n",
        "                            loss(output, target).item()\n",
        "                        )\n",
        "                        targets.extend(batch[\"target\"].numpy())\n",
        "                        predictions.extend(output.argmax(axis=1).detach().cpu().numpy())\n",
        "                    mlflow.log_metric(\"validation_loss\", sum(running_loss) / len(running_loss), epoch)\n",
        "                    mlflow.log_metric(\"validation_bacc\", balanced_accuracy_score(targets, predictions), epoch)\n",
        "\n",
        "        if test_dataset:\n",
        "            logging.info(\"Evaluating model on test\")\n",
        "            model.eval()\n",
        "            running_loss = []\n",
        "            targets = []\n",
        "            predictions = []\n",
        "            with torch.no_grad():\n",
        "                for batch in tqdm(test_loader):\n",
        "                    data = batch[\"data\"].to(device)\n",
        "                    target = batch[\"target\"].to(device)\n",
        "                    output = model(data)\n",
        "                    running_loss.append(\n",
        "                        loss(output, target).item()\n",
        "                    )\n",
        "                    targets.extend(batch[\"target\"].numpy())\n",
        "                    predictions.extend(output.argmax(axis=1).detach().cpu().numpy())\n",
        "                mlflow.log_metric(\"test_loss\", sum(running_loss) / len(running_loss), epoch)\n",
        "                mlflow.log_metric(\"test_bacc\", balanced_accuracy_score(targets, predictions), epoch)\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}
# Conv1d takes (batch, channel, seq_len)\n", #max conv\n", # This can be a hypterparameter\n", # This can be a hyperparameter\n", # Log all relevent hyperparameters\n", # This can be a hyperparameter\n", # This can be a hyperparameter\n", # This can be a hyperparameter\n",