{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "X4cRE8IbIrIV" }, "source": [ "If you're opening this Notebook on colab, you will probably need to install πŸ€— Transformers and πŸ€— Datasets. Uncomment the following cell and run it." ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "id": "MOsHUjgdIrIW" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting datasets\n", " Downloading datasets-2.8.0-py3-none-any.whl (452 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m452.9/452.9 kB\u001b[0m \u001b[31m56.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting transformers\n", " Downloading transformers-4.25.1-py3-none-any.whl (5.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m140.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting multiprocess\n", " Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 kB\u001b[0m \u001b[31m40.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: packaging in ./.local/lib/python3.8/site-packages (from datasets) (21.3)\n", "Requirement already satisfied: numpy>=1.17 in ./.local/lib/python3.8/site-packages (from datasets) (1.23.4)\n", "Collecting huggingface-hub<1.0.0,>=0.2.0\n", " Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.4/182.4 kB\u001b[0m \u001b[31m48.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting responses<0.19\n", " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", "Collecting xxhash\n", " Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 kB\u001b[0m \u001b[31m55.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in ./.local/lib/python3.8/site-packages (from datasets) (2.28.1)\n", "Collecting dill<0.3.7\n", " Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m35.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting aiohttp\n", " Downloading aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m113.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting fsspec[http]>=2021.11.1\n", " Downloading fsspec-2022.11.0-py3-none-any.whl (139 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.5/139.5 kB\u001b[0m \u001b[31m37.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in ./.local/lib/python3.8/site-packages (from datasets) (4.64.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets) (5.3.1)\n", "Requirement already satisfied: pandas in ./.local/lib/python3.8/site-packages (from datasets) (1.5.1)\n", "Collecting pyarrow>=6.0.0\n", " Downloading pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (36.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.0/36.0 MB\u001b[0m \u001b[31m88.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n", " Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m141.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", "\u001b[?25hCollecting regex!=2019.12.17\n", " Downloading regex-2022.10.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (772 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m772.3/772.3 kB\u001b[0m \u001b[31m107.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers) (3.0.12)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets) (2.1.1)\n", "Collecting async-timeout<5.0,>=4.0.0a3\n", " Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n", "Collecting aiosignal>=1.1.2\n", " Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n", "Collecting multidict<7.0,>=4.5\n", " Downloading multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.3/121.3 kB\u001b[0m \u001b[31m35.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting yarl<2.0,>=1.0\n", " Downloading yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (262 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m262.1/262.1 kB\u001b[0m \u001b[31m55.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting frozenlist>=1.1.1\n", " Downloading frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (161 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m161.3/161.3 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->datasets) (19.3.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.local/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.4.0)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/lib/python3/dist-packages (from packaging->datasets) (2.4.6)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2.8)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2019.11.28)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (1.25.8)\n", "Collecting urllib3<1.27,>=1.21.1\n", " Downloading urllib3-1.26.13-py2.py3-none-any.whl (140 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 kB\u001b[0m \u001b[31m40.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in ./.local/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in ./.local/lib/python3.8/site-packages (from pandas->datasets) (2022.5)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.14.0)\n", "Installing collected packages: tokenizers, xxhash, urllib3, regex, pyarrow, multidict, fsspec, frozenlist, dill, async-timeout, yarl, multiprocess, aiosignal, responses, huggingface-hub, aiohttp, transformers, datasets\n", "Successfully installed aiohttp-3.8.3 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.8.0 dill-0.3.6 frozenlist-1.3.3 fsspec-2022.11.0 huggingface-hub-0.11.1 multidict-6.0.4 multiprocess-0.70.14 pyarrow-10.0.1 regex-2022.10.31 responses-0.18.0 tokenizers-0.13.2 transformers-4.25.1 urllib3-1.26.13 xxhash-3.2.0 yarl-1.8.2\n", "\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m22.3.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n" ] } ], "source": [ "! pip install datasets transformers" ] }, { "cell_type": "markdown", "metadata": { "id": "oc3pMkfOvSzY" }, "source": [ "If you're opening this notebook locally, make sure your environment has an install from the last version of those libraries.\n", "\n", "To be able to share your model with the community and generate results like the one shown in the picture below via the inference API, there are a few more steps to follow.\n", "\n", "First you have to store your authentication token from the Hugging Face website (sign up [here](https://huggingface.co/join) if you haven't already!) then execute the following cell and input your username and password:" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "id": "lWbvUuN3vSzZ" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "04c95803576244b6bc7cf04dd300c67f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HTML(value='
\n", " \n", " \n", " \n", " id\n", " title\n", " context\n", " question\n", " answers\n", " \n", " \n", " \n", " \n", " 0\n", " 570c2257ec8fbc190045bc63\n", " Antarctica\n", " Antarctica, on average, is the coldest, driest, and windiest continent, and has the highest average elevation of all the continents. Antarctica is considered a desert, with annual precipitation of only 200 mm (8 in) along the coast and far less inland. The temperature in Antarctica has reached βˆ’89.2 Β°C (βˆ’128.6 Β°F), though the average for the third quarter (the coldest part of the year) is βˆ’63 Β°C (βˆ’81 Β°F). There are no permanent human residents, but anywhere from 1,000 to 5,000 people reside throughout the year at the research stations scattered across the continent. Organisms native to Antarctica include many types of algae, bacteria, fungi, plants, protista, and certain animals, such as mites, nematodes, penguins, seals and tardigrades. Vegetation, where it occurs, is tundra.\n", " What is Antarctica's annual precipitation along the coast?\n", " {'text': ['200 mm (8 in)'], 'answer_start': [202]}\n", " \n", " \n", " 1\n", " 56f8d30f9e9bad19000a05a5\n", " Brain\n", " Once a neuron is in place, it extends dendrites and an axon into the area around it. Axons, because they commonly extend a great distance from the cell body and need to reach specific targets, grow in a particularly complex way. The tip of a growing axon consists of a blob of protoplasm called a growth cone, studded with chemical receptors. These receptors sense the local environment, causing the growth cone to be attracted or repelled by various cellular elements, and thus to be pulled in a particular direction at each point along its path. The result of this pathfinding process is that the growth cone navigates through the brain until it reaches its destination area, where other chemical cues cause it to begin generating synapses. Considering the entire brain, thousands of genes create products that influence axonal pathfinding.\n", " What two structures does a neuron extend when it is in place during development?\n", " {'text': ['dendrites and an axon'], 'answer_start': [38]}\n", " \n", " \n", " 2\n", " 572654ddf1498d1400e8dc3d\n", " Department_store\n", " Marshall Field & Company originated in 1852. It was the premier department store on the main shopping street in the Midwest, State Street in Chicago. Upscale shoppers came by train from throughout the region, patronizing nearby hotels. It grew to become a major chain before converting to the Macy's nameplate on 9 September 2006. Marshall Field's Served as a model for other departments stores in that it had exceptional customer service. Field's also brought with it the now famous Frango mints brand that became so closely identified with Marshall Field's and Chicago from the now defunct Frederick & Nelson Department store. Marshall Field's also had the firsts, among many innovations by Marshall Field's. Field's had the first European buying office, which was located in Manchester, England, and the first bridal registry. The company was the first to introduce the concept of the personal shopper, and that service was provided without charge in every Field's store, until the chain's last days under the Marshall Field's name. It was the first store to offer revolving credit and the first department store to use escalators. Marshall Field's book department in the State Street store was legendary; it pioneered the concept of the \"book signing.\" Moreover, every year at Christmas, Marshall Field's downtown store windows were filled with animated displays as part of the downtown shopping district display; the \"theme\" window displays became famous for their ingenuity and beauty, and visiting the Marshall Field's windows at Christmas became a tradition for Chicagoans and visitors alike, as popular a local practice as visiting the Walnut Room with its equally famous Christmas tree or meeting \"under the clock\" on State Street.\n", " When did Marshall's convert to the Macy's name?\n", " {'text': ['9 September 2006'], 'answer_start': [313]}\n", " \n", " \n", " 3\n", " 5727b5944b864d1900163afa\n", " Switzerland\n", " Swiss citizens are universally required to buy health insurance from private insurance companies, which in turn are required to accept every applicant. While the cost of the system is among the highest it compares well with other European countries in terms of health outcomes; patients who are citizens have been reported as being, in general, highly satisfied with it. In 2012, life expectancy at birth was 80.4 years for men and 84.7 years for women β€” the highest in the world. However, spending on health is particularly high at 11.4% of GDP (2010), on par with Germany and France (11.6%) and other European countries, and notably less than spending in the USA (17.6%). From 1990, a steady increase can be observed, reflecting the high costs of the services provided. With an ageing population and new healthcare technologies, health spending will likely continue to rise.\n", " In 2012, what was Switzerland's world ranking for life expectancy in 2012?\n", " {'text': ['highest'], 'answer_start': [459]}\n", " \n", " \n", " 4\n", " 572910176aef051400154a10\n", " Software_testing\n", " A primary purpose of testing is to detect software failures so that defects may be discovered and corrected. Testing cannot establish that a product functions properly under all conditions but can only establish that it does not function properly under specific conditions. The scope of software testing often includes examination of code as well as execution of that code in various environments and conditions as well as examining the aspects of code: does it do what it is supposed to do and do what it needs to do. In the current culture of software development, a testing organization may be separate from the development team. There are various roles for testing team members. Information derived from software testing may be used to correct the process by which software is developed.\n", " What does the scope of testing the software also look at?\n", " {'text': ['examination of code as well as execution of that code'], 'answer_start': [319]}\n", " \n", " \n", " 5\n", " 57307593396df91900096125\n", " Translation\n", " Relying exclusively on unedited machine translation, however, ignores the fact that communication in human language is context-embedded and that it takes a person to comprehend the context of the original text with a reasonable degree of probability. It is certainly true that even purely human-generated translations are prone to error; therefore, to ensure that a machine-generated translation will be useful to a human being and that publishable-quality translation is achieved, such translations must be reviewed and edited by a human.\n", " How must machine translations be transformed by a human?\n", " {'text': ['reviewed and edited'], 'answer_start': [508]}\n", " \n", " \n", " 6\n", " 5727cc3a3acd2414000deca2\n", " Detroit\n", " Precipitation is moderate and somewhat evenly distributed throughout the year, although the warmer months such as May and June average more, averaging 33.5 inches (850 mm) annually, but historically ranging from 20.49 in (520 mm) in 1963 to 47.70 in (1,212 mm) in 2011. Snowfall, which typically falls in measurable amounts between November 15 through April 4 (occasionally in October and very rarely in May), averages 42.5 inches (108 cm) per season, although historically ranging from 11.5 in (29 cm) in 1881βˆ’82 to 94.9 in (241 cm) in 2013βˆ’14. A thick snowpack is not often seen, with an average of only 27.5 days with 3 in (7.6 cm) or more of snow cover. Thunderstorms are frequent in the Detroit area. These usually occur during spring and summer.\n", " How many inches of snow does Detroit get on average?\n", " {'text': ['42.5'], 'answer_start': [419]}\n", " \n", " \n", " 7\n", " 5725b82838643c19005acbc6\n", " Montevideo\n", " A few years after its foundation, Montevideo became the main city of the region north of the RΓ­o de la Plata and east of the Uruguay River, competing with Buenos Aires for dominance in maritime commerce. The importance of Montevideo as the main port of the Viceroyalty of the RΓ­o de la Plata brought it in confrontations with the city of Buenos Aires in various occasions, including several times when it was taken over to be used as a base to defend the eastern province of the Viceroyalty from Portuguese incursions.\n", " What were Buenos Aires and Montevideo fighting for dominance over?\n", " {'text': ['maritime commerce'], 'answer_start': [185]}\n", " \n", " \n", " 8\n", " 572928341d046914007790e2\n", " Planck_constant\n", " Prior to Planck's work, it had been assumed that the energy of a body could take on any value whatsoever – that it was a continuous variable. The Rayleigh–Jeans law makes close predictions for a narrow range of values at one limit of temperatures, but the results diverge more and more strongly as temperatures increase. To make Planck's law, which correctly predicts blackbody emissions, it was necessary to multiply the classical expression by a complex factor that involves h in both the numerator and the denominator. The influence of h in this complex factor would not disappear if it were set to zero or to any other value. Making an equation out of Planck's law that would reproduce the Rayleigh–Jeans law could not be done by changing the values of h, of the Boltzmann constant, or of any other constant or variable in the equation. In this case the picture given by classical physics is not duplicated by a range of results in the quantum picture.\n", " The Rayleigh-Jeans law makes close predictions for what amount of values?\n", " {'text': ['a narrow range'], 'answer_start': [193]}\n", " \n", " \n", " 9\n", " 570da68e16d0071400510c4c\n", " Antarctica\n", " On 6 September 2007, Belgian-based International Polar Foundation unveiled the Princess Elisabeth station, the world's first zero-emissions polar science station in Antarctica to research climate change. Costing $16.3 million, the prefabricated station, which is part of the International Polar Year, was shipped to the South Pole from Belgium by the end of 2008 to monitor the health of the polar regions. Belgian polar explorer Alain Hubert stated: \"This base will be the first of its kind to produce zero emissions, making it a unique model of how energy should be used in the Antarctic.\" Johan Berte is the leader of the station design team and manager of the project which conducts research in climatology, glaciology and microbiology.\n", " How much did the Princess Elizabeth station cost?\n", " {'text': ['$16.3 million'], 'answer_start': [212]}\n", " \n", " \n", "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "show_random_elements(datasets[\"train\"])" ] }, { "cell_type": "markdown", "metadata": { "id": "n9qywopnIrJH" }, "source": [ "## Preprocessing the training data" ] }, { "cell_type": "markdown", "metadata": { "id": "YVx71GdAIrJH" }, "source": [ "Before we can feed those texts to our model, we need to preprocess them. This is done by a πŸ€— Transformers `Tokenizer` which will (as the name indicates) tokenize the inputs (including converting the tokens to their corresponding IDs in the pretrained vocabulary) and put it in a format the model expects, as well as generate the other inputs that model requires.\n", "\n", "To do all of this, we instantiate our tokenizer with the `AutoTokenizer.from_pretrained` method, which will ensure:\n", "\n", "- we get a tokenizer that corresponds to the model architecture we want to use,\n", "- we download the vocabulary used when pretraining this specific checkpoint.\n", "\n", "That vocabulary will be cached, so it's not downloaded again the next time we run the cell." ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "id": "eXNLu_-nIrJI" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c1933915e10843d883eab552a2fa1302", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Downloading: 0%| | 0.00/28.0 [00:00 384:\n", " break\n", "example = datasets[\"train\"][i]" ] }, { "cell_type": "markdown", "metadata": { "id": "YqBE60lXvSzl" }, "source": [ "Without any truncation, we get the following length for the input IDs:" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "id": "Oe5cLyLYvSzl", "outputId": "98292ffb-17ee-48ac-e9ff-d82bf13f4a22" }, "outputs": [ { "data": { "text/plain": [ "396" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(tokenizer(example[\"question\"], example[\"context\"])[\"input_ids\"])" ] }, { "cell_type": "markdown", "metadata": { "id": "dka9MlZWvSzl" }, "source": [ "Now, if we just truncate, we will lose information (and possibly the answer to our question):" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "id": "yLGSVJaEvSzl", "outputId": "b8702093-03ca-473e-9b03-876862dd8a61" }, "outputs": [ { "data": { "text/plain": [ "384" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(tokenizer(example[\"question\"], example[\"context\"], max_length=max_length, truncation=\"only_second\")[\"input_ids\"])" ] }, { "cell_type": "markdown", "metadata": { "id": "pMOGlhp-vSzm" }, "source": [ "Note that we never want to truncate the question, only the context, else the `only_second` truncation picked. Now, our tokenizer can automatically return us a list of features capped by a certain maximum length, with the overlap we talked above, we just have to tell it with `return_overflowing_tokens=True` and by passing the stride:" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "id": "PdlmMEwFvSzm" }, "outputs": [], "source": [ "tokenized_example = tokenizer(\n", " example[\"question\"],\n", " example[\"context\"],\n", " max_length=max_length,\n", " truncation=\"only_second\",\n", " return_overflowing_tokens=True,\n", " stride=doc_stride\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "0ppTIYLMvSzm" }, "source": [ "Now we don't have one list of `input_ids`, but several: " ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "id": "l1TYNC60vSzm", "outputId": "55e10f52-387a-42c3-e835-f12edb5cdbcd" }, "outputs": [ { "data": { "text/plain": [ "[384, 157]" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[len(x) for x in tokenized_example[\"input_ids\"]]" ] }, { "cell_type": "markdown", "metadata": { "id": "PgawQA-7vSzn" }, "source": [ "And if we decode them, we can see the overlap:" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "id": "3rapuR04vSzn", "outputId": "84dd4fa0-dab9-4eba-8fb5-d0bd61c791a3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[CLS] how many wins does the notre dame men's basketball team have? [SEP] the men's basketball team has over 1, 600 wins, one of only 12 schools who have reached that mark, and have appeared in 28 ncaa tournaments. former player austin carr holds the record for most points scored in a single game of the tournament with 61. although the team has never won the ncaa tournament, they were named by the helms athletic foundation as national champions twice. the team has orchestrated a number of upsets of number one ranked teams, the most notable of which was ending ucla's record 88 - game winning streak in 1974. the team has beaten an additional eight number - one teams, and those nine wins rank second, to ucla's 10, all - time in wins against the top team. the team plays in newly renovated purcell pavilion ( within the edmund p. joyce center ), which reopened for the beginning of the 2009 – 2010 season. the team is coached by mike brey, who, as of the 2014 – 15 season, his fifteenth at notre dame, has achieved a 332 - 165 record. in 2009 they were invited to the nit, where they advanced to the semifinals but were beaten by penn state who went on and beat baylor in the championship. the 2010 – 11 team concluded its regular season ranked number seven in the country, with a record of 25 – 5, brey's fifth straight 20 - win season, and a second - place finish in the big east. during the 2014 - 15 season, the team went 32 - 6 and won the acc conference tournament, later advancing to the elite 8, where the fighting irish lost on a missed buzzer - beater against then undefeated kentucky. led by nba draft picks jerian grant and pat connaughton, the fighting irish beat the eventual national champion duke blue devils twice during the season. the 32 wins were [SEP]\n", "[CLS] how many wins does the notre dame men's basketball team have? [SEP] championship. the 2010 – 11 team concluded its regular season ranked number seven in the country, with a record of 25 – 5, brey's fifth straight 20 - win season, and a second - place finish in the big east. during the 2014 - 15 season, the team went 32 - 6 and won the acc conference tournament, later advancing to the elite 8, where the fighting irish lost on a missed buzzer - beater against then undefeated kentucky. led by nba draft picks jerian grant and pat connaughton, the fighting irish beat the eventual national champion duke blue devils twice during the season. the 32 wins were the most by the fighting irish team since 1908 - 09. [SEP]\n" ] } ], "source": [ "for x in tokenized_example[\"input_ids\"][:2]:\n", " print(tokenizer.decode(x))" ] }, { "cell_type": "markdown", "metadata": { "id": "Cw7JzXusvSzn" }, "source": [ "Now this will give us some work to properly treat the answers: we need to find in which of those features the answer actually is, and where exactly in that feature. The models we will use require the start and end positions of these answers in the tokens, so we will also need to to map parts of the original context to some tokens. Thankfully, the tokenizer we're using can help us with that by returning an `offset_mapping`:" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "id": "_yEbtSTyvSzn", "outputId": "4a1ddf42-5751-4383-8d31-b8728ff67a18" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[(0, 0), (0, 3), (4, 8), (9, 13), (14, 18), (19, 22), (23, 28), (29, 33), (34, 37), (37, 38), (38, 39), (40, 50), (51, 55), (56, 60), (60, 61), (0, 0), (0, 3), (4, 7), (7, 8), (8, 9), (10, 20), (21, 25), (26, 29), (30, 34), (35, 36), (36, 37), (37, 40), (41, 45), (45, 46), (47, 50), (51, 53), (54, 58), (59, 61), (62, 69), (70, 73), (74, 78), (79, 86), (87, 91), (92, 96), (96, 97), (98, 101), (102, 106), (107, 115), (116, 118), (119, 121), (122, 126), (127, 138), (138, 139), (140, 146), (147, 153), (154, 160), (161, 165), (166, 171), (172, 175), (176, 182), (183, 186), (187, 191), (192, 198), (199, 205), (206, 208), (209, 210), (211, 217), (218, 222), (223, 225), (226, 229), (230, 240), (241, 245), (246, 248), (248, 249), (250, 258), (259, 262), (263, 267), (268, 271), (272, 277), (278, 281), (282, 285), (286, 290), (291, 301), (301, 302), (303, 307), (308, 312), (313, 318), (319, 321), (322, 325), (326, 330), (330, 331), (332, 340), (341, 351), (352, 354), (355, 363), (364, 373), (374, 379), (379, 380), (381, 384), (385, 389), (390, 393), (394, 406), (407, 408), (409, 415), (416, 418)]\n" ] } ], "source": [ "tokenized_example = tokenizer(\n", " example[\"question\"],\n", " example[\"context\"],\n", " max_length=max_length,\n", " truncation=\"only_second\",\n", " return_overflowing_tokens=True,\n", " return_offsets_mapping=True,\n", " stride=doc_stride\n", ")\n", "print(tokenized_example[\"offset_mapping\"][0][:100])" ] }, { "cell_type": "markdown", "metadata": { "id": "evsNSwXrvSzo" }, "source": [ "This gives, for each index of our input IDS, the corresponding start and end character in the original text that gave our token. The very first token (`[CLS]`) has (0, 0) because it doesn't correspond to any part of the question/answer, then the second token is the same as the characters 0 to 3 of the question:" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "id": "zcoV2__vvSzo", "outputId": "b9d2aa85-baac-4f10-b13a-755ac00e4b58" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "how How\n" ] } ], "source": [ "first_token_id = tokenized_example[\"input_ids\"][0][1]\n", "offsets = tokenized_example[\"offset_mapping\"][0][1]\n", "print(tokenizer.convert_ids_to_tokens([first_token_id])[0], example[\"question\"][offsets[0]:offsets[1]])" ] }, { "cell_type": "markdown", "metadata": { "id": "sCrbTA35vSzo" }, "source": [ "So we can use this mapping to find the position of the start and end tokens of our answer in a given feature. We just have to distinguish which parts of the offsets correspond to the question and which part correspond to the context, this is where the `sequence_ids` method of our `tokenized_example` can be useful:" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "id": "dBJF4HGUvSzo", "outputId": "11d12297-036d-47fe-8f17-feaad6f0c905" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n" ] } ], "source": [ "sequence_ids = tokenized_example.sequence_ids()\n", "print(sequence_ids)" ] }, { "cell_type": "markdown", "metadata": { "id": "PG8V3Xh-vSzp" }, "source": [ "It returns `None` for the special tokens, then 0 or 1 depending on whether the corresponding token comes from the first sentence past (the question) or the second (the context). Now with all of this, we can find the first and last token of the answer in one of our input feature (or if the answer is not in this feature):" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "id": "OXH3Ee38vSzp", "outputId": "3e0479c2-5f80-49ed-c895-b6d7034c446c" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "23 26\n" ] } ], "source": [ "answers = example[\"answers\"]\n", "start_char = answers[\"answer_start\"][0]\n", "end_char = start_char + len(answers[\"text\"][0])\n", "\n", "# Start token index of the current span in the text.\n", "token_start_index = 0\n", "while sequence_ids[token_start_index] != 1:\n", " token_start_index += 1\n", "\n", "# End token index of the current span in the text.\n", "token_end_index = len(tokenized_example[\"input_ids\"][0]) - 1\n", "while sequence_ids[token_end_index] != 1:\n", " token_end_index -= 1\n", "\n", "# Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n", "offsets = tokenized_example[\"offset_mapping\"][0]\n", "if (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n", " # Move the token_start_index and token_end_index to the two ends of the answer.\n", " # Note: we could go after the last offset if the answer is the last word (edge case).\n", " while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n", " token_start_index += 1\n", " start_position = token_start_index - 1\n", " while offsets[token_end_index][1] >= end_char:\n", " token_end_index -= 1\n", " end_position = token_end_index + 1\n", " print(start_position, end_position)\n", "else:\n", " print(\"The answer is not in this feature.\")" ] }, { "cell_type": "markdown", "metadata": { "id": "n4j1fRwPvSzp" }, "source": [ "And we can double check that it is indeed the theoretical answer:" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "id": "51ghsvoqvSzp", "outputId": "e5bf4a9d-f7c4-43e2-968f-3ea873392190" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "over 1, 600\n", "over 1,600\n" ] } ], "source": [ "print(tokenizer.decode(tokenized_example[\"input_ids\"][0][start_position: end_position+1]))\n", "print(answers[\"text\"][0])" ] }, { "cell_type": "markdown", "metadata": { "id": "7UGRp61cvSzp" }, "source": [ "For this notebook to work with any kind of models, we need to account for the special case where the model expects padding on the left (in which case we switch the order of the question and the context):" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "id": "S4nPL0O0vSzq" }, "outputs": [], "source": [ "pad_on_right = tokenizer.padding_side == \"right\"" ] }, { "cell_type": "markdown", "metadata": { "id": "jOpXTOrBvSzq" }, "source": [ "Now let's put everything together in one function we will apply to our training set. In the case of impossible answers (the answer is in another feature given by an example with a long context), we set the cls index for both the start and end position. We could also simply discard those examples from the training set if the flag `allow_impossible_answers` is `False`. Since the preprocessing is already complex enough as it is, we've kept is simple for this part." ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "id": "VbTQuHtGvSzq" }, "outputs": [], "source": [ "def prepare_train_features(examples):\n", " # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n", " # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n", " # left whitespace\n", " examples[\"question\"] = [q.lstrip() for q in examples[\"question\"]]\n", "\n", " # Tokenize our examples with truncation and padding, but keep the overflows using a stride. This results\n", " # in one example possible giving several features when a context is long, each of those features having a\n", " # context that overlaps a bit the context of the previous feature.\n", " tokenized_examples = tokenizer(\n", " examples[\"question\" if pad_on_right else \"context\"],\n", " examples[\"context\" if pad_on_right else \"question\"],\n", " truncation=\"only_second\" if pad_on_right else \"only_first\",\n", " max_length=max_length,\n", " stride=doc_stride,\n", " return_overflowing_tokens=True,\n", " return_offsets_mapping=True,\n", " padding=\"max_length\",\n", " )\n", "\n", " # Since one example might give us several features if it has a long context, we need a map from a feature to\n", " # its corresponding example. This key gives us just that.\n", " sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n", " # The offset mappings will give us a map from token to character position in the original context. This will\n", " # help us compute the start_positions and end_positions.\n", " offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n", "\n", " # Let's label those examples!\n", " tokenized_examples[\"start_positions\"] = []\n", " tokenized_examples[\"end_positions\"] = []\n", "\n", " for i, offsets in enumerate(offset_mapping):\n", " # We will label impossible answers with the index of the CLS token.\n", " input_ids = tokenized_examples[\"input_ids\"][i]\n", " cls_index = input_ids.index(tokenizer.cls_token_id)\n", "\n", " # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n", " sequence_ids = tokenized_examples.sequence_ids(i)\n", "\n", " # One example can give several spans, this is the index of the example containing this span of text.\n", " sample_index = sample_mapping[i]\n", " answers = examples[\"answers\"][sample_index]\n", " # If no answers are given, set the cls_index as answer.\n", " if len(answers[\"answer_start\"]) == 0:\n", " tokenized_examples[\"start_positions\"].append(cls_index)\n", " tokenized_examples[\"end_positions\"].append(cls_index)\n", " else:\n", " # Start/end character index of the answer in the text.\n", " start_char = answers[\"answer_start\"][0]\n", " end_char = start_char + len(answers[\"text\"][0])\n", "\n", " # Start token index of the current span in the text.\n", " token_start_index = 0\n", " while sequence_ids[token_start_index] != (1 if pad_on_right else 0):\n", " token_start_index += 1\n", "\n", " # End token index of the current span in the text.\n", " token_end_index = len(input_ids) - 1\n", " while sequence_ids[token_end_index] != (1 if pad_on_right else 0):\n", " token_end_index -= 1\n", "\n", " # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n", " if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n", " tokenized_examples[\"start_positions\"].append(cls_index)\n", " tokenized_examples[\"end_positions\"].append(cls_index)\n", " else:\n", " # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n", " # Note: we could go after the last offset if the answer is the last word (edge case).\n", " while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n", " token_start_index += 1\n", " tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n", " while offsets[token_end_index][1] >= end_char:\n", " token_end_index -= 1\n", " tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n", "\n", " return tokenized_examples" ] }, { "cell_type": "markdown", "metadata": { "id": "0lm8ozrJIrJR" }, "source": [ "This function works with one or several examples. In the case of several examples, the tokenizer will return a list of lists for each key:" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "id": "-b70jh26IrJS" }, "outputs": [], "source": [ "features = prepare_train_features(datasets['train'][:5])" ] }, { "cell_type": "markdown", "metadata": { "id": "zS-6iXTkIrJT" }, "source": [ "To apply this function on all the sentences (or pairs of sentences) in our dataset, we just use the `map` method of our `dataset` object we created earlier. This will apply the function on all the elements of all the splits in `dataset`, so our training, validation and testing data will be preprocessed in one single command. Since our preprocessing changes the number of samples, we need to remove the old columns when applying it." ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "id": "DDtsaJeVIrJT", "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "95736c0d99994e25b21cfa8330d4655b", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/88 [00:00\n", " \n", " \n", " [16599/16599 55:46, Epoch 3/3]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss
11.2196001.180651
20.9562001.121280
30.7477001.164216

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-1000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-1000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-1000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-1000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-1000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-1500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-1500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-1500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-1500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-1500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-2000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-2000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-2000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-2000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-2000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-2500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-2500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-2500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-2500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-2500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-3000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-3000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-3000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-3000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-3000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-3500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-3500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-3500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-3500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-3500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-4000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-4000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-4000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-4000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-4000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-4500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-4500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-4500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-4500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-4500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-5000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-5000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-5000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-5000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-5000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-5500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-5500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-5500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-5500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-5500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "***** Running Evaluation *****\n", " Num examples = 10784\n", " Batch size = 16\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-6000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-6000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-6000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-6000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-6000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-6500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-6500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-6500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-6500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-6500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-7000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-7000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-7000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-7000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-7000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-7500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-7500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-7500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-7500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-7500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-8000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-8000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-8000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-8000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-8000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-8500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-8500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-8500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-8500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-8500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-9000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-9000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-9000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-9000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-9000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-9500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-9500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-9500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-9500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-9500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-10000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-10000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-10000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-10000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-10000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-10500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-10500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-10500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-10500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-10500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-11000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-11000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-11000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-11000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-11000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "***** Running Evaluation *****\n", " Num examples = 10784\n", " Batch size = 16\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-11500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-11500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-11500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-11500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-11500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-12000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-12000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-12000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-12000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-12000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-12500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-12500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-12500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-12500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-12500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-13000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-13000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-13000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-13000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-13000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-13500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-13500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-13500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-13500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-13500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-14000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-14000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-14000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-14000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-14000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-14500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-14500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-14500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-14500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-14500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-15000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-15000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-15000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-15000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-15000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-15500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-15500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-15500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-15500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-15500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-16000\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-16000/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-16000/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-16000/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-16000/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad/checkpoint-16500\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/checkpoint-16500/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/checkpoint-16500/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/checkpoint-16500/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/checkpoint-16500/special_tokens_map.json\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n", "***** Running Evaluation *****\n", " Num examples = 10784\n", " Batch size = 16\n", "\n", "\n", "Training completed. Do not forget to share your model on huggingface.co/models =)\n", "\n", "\n" ] }, { "data": { "text/plain": [ "TrainOutput(global_step=16599, training_loss=1.0824026910646385, metrics={'train_runtime': 3347.5605, 'train_samples_per_second': 79.333, 'train_steps_per_second': 4.959, 'total_flos': 2.602335381127373e+16, 'train_loss': 1.0824026910646385, 'epoch': 3.0})" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "trainer.train()" ] }, { "cell_type": "markdown", "metadata": { "id": "9osNI3s-vSzt" }, "source": [ "Since this training is particularly long, let's save the model just in case we need to restart." ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "id": "QcqlZ1NjvSzt" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to test-squad-trained\n", "Configuration saved in test-squad-trained/config.json\n", "Model weights saved in test-squad-trained/pytorch_model.bin\n", "tokenizer config file saved in test-squad-trained/tokenizer_config.json\n", "Special tokens file saved in test-squad-trained/special_tokens_map.json\n", "Saving model checkpoint to distilbert-base-uncased-finetuned-squad\n", "Configuration saved in distilbert-base-uncased-finetuned-squad/config.json\n", "Model weights saved in distilbert-base-uncased-finetuned-squad/pytorch_model.bin\n", "tokenizer config file saved in distilbert-base-uncased-finetuned-squad/tokenizer_config.json\n", "Special tokens file saved in distilbert-base-uncased-finetuned-squad/special_tokens_map.json\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "71026a6a2a9c4467a36c1d364294c629", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload file pytorch_model.bin: 0%| | 32.0k/253M [00:00 main\n", "\n", "Dropping the following result as it does not have all the necessary fields:\n", "{'task': {'name': 'Question Answering', 'type': 'question-answering'}, 'dataset': {'name': 'squad', 'type': 'squad', 'config': 'plain_text', 'split': 'train', 'args': 'plain_text'}}\n", "To https://huggingface.co/MMars/distilbert-base-uncased-finetuned-squad\n", " 23f99fe..9ce7a49 main -> main\n", "\n" ] } ], "source": [ "trainer.save_model(\"test-squad-trained\")" ] }, { "cell_type": "markdown", "metadata": { "id": "JP6CTR_-vSzt" }, "source": [ "## Evaluation" ] }, { "cell_type": "markdown", "metadata": { "id": "sit807DcvSzt" }, "source": [ "Evaluating our model will require a bit more work, as we will need to map the predictions of our model back to parts of the context. The model itself predicts logits for the start and en position of our answers: if we take a batch from our validation datalaoder, here is the output our model gives us:" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "id": "8aFs9GAqvSzt", "outputId": "ba3db509-9506-4be2-969e-12c0b352902d" }, "outputs": [ { "data": { "text/plain": [ "odict_keys(['loss', 'start_logits', 'end_logits'])" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import torch\n", "\n", "for batch in trainer.get_eval_dataloader():\n", " break\n", "batch = {k: v.to(trainer.args.device) for k, v in batch.items()}\n", "with torch.no_grad():\n", " output = trainer.model(**batch)\n", "output.keys()" ] }, { "cell_type": "markdown", "metadata": { "id": "FagCzTEsvSzt" }, "source": [ "The output of the model is a dict-like object that contains the loss (since we provided labels), the start and end logits. We won't need the loss for our predictions, let's have a look a the logits:" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "id": "swutB426vSzu", "outputId": "b61d651f-7aea-4cd9-a703-0276bd02f901" }, "outputs": [ { "data": { "text/plain": [ "(torch.Size([16, 384]), torch.Size([16, 384]))" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "output.start_logits.shape, output.end_logits.shape" ] }, { "cell_type": "markdown", "metadata": { "id": "M8ro0Lf3vSzu" }, "source": [ "We have one logit for each feature and each token. The most obvious thing to predict an answer for each featyre is to take the index for the maximum of the start logits as a start position and the index of the maximum of the end logits as an end position." ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "id": "XoJrnuc_vSzu", "outputId": "c168f296-2a6f-4fbd-d7cc-f2cde4913057" }, "outputs": [ { "data": { "text/plain": [ "(tensor([ 46, 57, 78, 54, 118, 107, 72, 35, 107, 34, 73, 41, 80, 91,\n", " 156, 35], device='cuda:0'),\n", " tensor([ 47, 58, 81, 44, 118, 110, 75, 37, 110, 36, 76, 42, 83, 94,\n", " 158, 35], device='cuda:0'))" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "output.start_logits.argmax(dim=-1), output.end_logits.argmax(dim=-1)" ] }, { "cell_type": "markdown", "metadata": { "id": "WGnoZRwuvSzu" }, "source": [ "This will work great in a lot of cases, but what if this prediction gives us something impossible: the start position could be greater than the end position, or point to a span of text in the question instead of the answer. In that case, we might want to look at the second best prediction to see if it gives a possible answer and select that instead.\n", "\n", "However, picking the second best answer is not as easy as picking the best one: is it the second best index in the start logits with the best index in the end logits? Or the best index in the start logits with the second best index in the end logits? And if that second best answer is not possible either, it gets even trickier for the third best answer.\n", "\n", "\n", "To classify our answers, we will use the score obtained by adding the start and end logits. We won't try to order all the possible answers and limit ourselves to with a hyper-parameter we call `n_best_size`. We'll pick the best indices in the start and end logits and gather all the answers this predicts. After checking if each one is valid, we will sort them by their score and keep the best one. Here is how we would do this on the first feature in the batch:" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "id": "sSG8WJ0fvSzu" }, "outputs": [], "source": [ "n_best_size = 20" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "id": "nqr98ymTvSzu" }, "outputs": [], "source": [ "import numpy as np\n", "\n", "start_logits = output.start_logits[0].cpu().numpy()\n", "end_logits = output.end_logits[0].cpu().numpy()\n", "# Gather the indices the best start/end logits:\n", "start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n", "end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n", "valid_answers = []\n", "for start_index in start_indexes:\n", " for end_index in end_indexes:\n", " if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n", " valid_answers.append(\n", " {\n", " \"score\": start_logits[start_index] + end_logits[end_index],\n", " \"text\": \"\" # We need to find a way to get back the original substring corresponding to the answer in the context\n", " }\n", " )" ] }, { "cell_type": "markdown", "metadata": { "id": "P4ObsoBtvSzu" }, "source": [ "And then we can sort the `valid_answers` according to their `score` and only keep the best one. The only point left is how to check a given span is inside the context (and not the question) and how to get back the text inside. To do this, we need to add two things to our validation features:\n", "- the ID of the example that generated the feature (since each example can generate several features, as seen before);\n", "- the offset mapping that will give us a map from token indices to character positions in the context.\n", "\n", "That's why we will re-process the validation set with the following function, slightly different from `prepare_train_features`:" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "id": "YAoSiv0yvSzv" }, "outputs": [], "source": [ "def prepare_validation_features(examples):\n", " # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n", " # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n", " # left whitespace\n", " examples[\"question\"] = [q.lstrip() for q in examples[\"question\"]]\n", "\n", " # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n", " # in one example possible giving several features when a context is long, each of those features having a\n", " # context that overlaps a bit the context of the previous feature.\n", " tokenized_examples = tokenizer(\n", " examples[\"question\" if pad_on_right else \"context\"],\n", " examples[\"context\" if pad_on_right else \"question\"],\n", " truncation=\"only_second\" if pad_on_right else \"only_first\",\n", " max_length=max_length,\n", " stride=doc_stride,\n", " return_overflowing_tokens=True,\n", " return_offsets_mapping=True,\n", " padding=\"max_length\",\n", " )\n", "\n", " # Since one example might give us several features if it has a long context, we need a map from a feature to\n", " # its corresponding example. This key gives us just that.\n", " sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n", "\n", " # We keep the example_id that gave us this feature and we will store the offset mappings.\n", " tokenized_examples[\"example_id\"] = []\n", "\n", " for i in range(len(tokenized_examples[\"input_ids\"])):\n", " # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n", " sequence_ids = tokenized_examples.sequence_ids(i)\n", " context_index = 1 if pad_on_right else 0\n", "\n", " # One example can give several spans, this is the index of the example containing this span of text.\n", " sample_index = sample_mapping[i]\n", " tokenized_examples[\"example_id\"].append(examples[\"id\"][sample_index])\n", "\n", " # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n", " # position is part of the context or not.\n", " tokenized_examples[\"offset_mapping\"][i] = [\n", " (o if sequence_ids[k] == context_index else None)\n", " for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n", " ]\n", "\n", " return tokenized_examples" ] }, { "cell_type": "markdown", "metadata": { "id": "NXN-qjJYvSzv" }, "source": [ "And like before, we can apply that function to our validation set easily:" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "colab": { "referenced_widgets": [ "32ba04d6240149f49eb48c8d8b7f9aae" ] }, "id": "37lPt_u2vSzv", "outputId": "2dcfc9a0-d017-4948-ba5c-4c527a8ec811" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5af7ce8baa5140c884caff5f7e47bbef", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/11 [00:00" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "raw_predictions = trainer.predict(validation_features)" ] }, { "cell_type": "markdown", "metadata": { "id": "wGtkWwtEvSzv" }, "source": [ "The `Trainer` *hides* the columns that are not used by the model (here `example_id` and `offset_mapping` which we will need for our post-processing), so we set them back:" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "id": "oK3_5QK6vSzv" }, "outputs": [], "source": [ "validation_features.set_format(type=validation_features.format[\"type\"], columns=list(validation_features.features.keys()))" ] }, { "cell_type": "markdown", "metadata": { "id": "Hd42yVT5vSzw" }, "source": [ "We can now refine the test we had before: since we set `None` in the offset mappings when it corresponds to a part of the question, it's easy to check if an answer is fully inside the context. We also eliminate very long answers from our considerations (with an hyper-parameter we can tune)" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "id": "N6NqUgGivSzw" }, "outputs": [], "source": [ "max_answer_length = 30" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "id": "aZ01s0RgvSzw", "outputId": "7332a0a3-d721-49b2-f7c0-eae95734b023" }, "outputs": [ { "data": { "text/plain": [ "[{'score': 15.539335, 'text': 'Denver Broncos'},\n", " {'score': 13.661739,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers'},\n", " {'score': 11.69728, 'text': 'Carolina Panthers'},\n", " {'score': 10.94569, 'text': 'Denver'},\n", " {'score': 10.793658, 'text': 'Broncos'},\n", " {'score': 8.916061,\n", " 'text': 'Broncos defeated the National Football Conference (NFC) champion Carolina Panthers'},\n", " {'score': 8.514549,\n", " 'text': 'The American Football Conference (AFC) champion Denver Broncos'},\n", " {'score': 8.262249,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10'},\n", " {'score': 8.133989,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC)'},\n", " {'score': 7.33582,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC) champion Carolina'},\n", " {'score': 7.2233815,\n", " 'text': 'Denver Broncos defeated the National Football Conference'},\n", " {'score': 6.6373134,\n", " 'text': 'American Football Conference (AFC) champion Denver Broncos'},\n", " {'score': 6.636953,\n", " 'text': 'The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers'},\n", " {'score': 6.4271107,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC'},\n", " {'score': 6.4092493,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC) champion'},\n", " {'score': 6.2977896, 'text': 'Carolina Panthers 24–10'},\n", " {'score': 5.9782686, 'text': 'Panthers'},\n", " {'score': 5.619999,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title.'},\n", " {'score': 5.3713613, 'text': 'Carolina'},\n", " {'score': 4.81654,\n", " 'text': 'Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title'}]" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "start_logits = output.start_logits[0].cpu().numpy()\n", "end_logits = output.end_logits[0].cpu().numpy()\n", "offset_mapping = validation_features[0][\"offset_mapping\"]\n", "# The first feature comes from the first example. For the more general case, we will need to be match the example_id to\n", "# an example index\n", "context = datasets[\"validation\"][0][\"context\"]\n", "\n", "# Gather the indices the best start/end logits:\n", "start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n", "end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n", "valid_answers = []\n", "for start_index in start_indexes:\n", " for end_index in end_indexes:\n", " # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n", " # to part of the input_ids that are not in the context.\n", " if (\n", " start_index >= len(offset_mapping)\n", " or end_index >= len(offset_mapping)\n", " or offset_mapping[start_index] is None\n", " or offset_mapping[end_index] is None\n", " ):\n", " continue\n", " # Don't consider answers with a length that is either < 0 or > max_answer_length.\n", " if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n", " continue\n", " if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n", " start_char = offset_mapping[start_index][0]\n", " end_char = offset_mapping[end_index][1]\n", " valid_answers.append(\n", " {\n", " \"score\": start_logits[start_index] + end_logits[end_index],\n", " \"text\": context[start_char: end_char]\n", " }\n", " )\n", "\n", "valid_answers = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[:n_best_size]\n", "valid_answers" ] }, { "cell_type": "markdown", "metadata": { "id": "6Icr4S0pvSzw" }, "source": [ "We can compare to the actual ground-truth answer:" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "id": "zlln3VSRvSzw", "outputId": "69c2f730-4f2b-4264-bfc3-5efc6326570e" }, "outputs": [ { "data": { "text/plain": [ "{'text': ['Denver Broncos', 'Denver Broncos', 'Denver Broncos'],\n", " 'answer_start': [177, 177, 177]}" ] }, "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ "datasets[\"validation\"][0][\"answers\"]" ] }, { "cell_type": "markdown", "metadata": { "id": "UGbF10qqvSzx" }, "source": [ "Our model picked the right as the most likely answer!\n", "\n", "As we mentioned in the code above, this was easy on the first feature because we knew it comes from the first example. For the other features, we will need a map between examples and their corresponding features. Also, since one example can give several features, we will need to gather together all the answers in all the features generated by a given example, then pick the best one. The following code builds a map from example index to its corresponding features indices:" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "id": "vJkdc-O6vSzx" }, "outputs": [], "source": [ "import collections\n", "\n", "examples = datasets[\"validation\"]\n", "features = validation_features\n", "\n", "example_id_to_index = {k: i for i, k in enumerate(examples[\"id\"])}\n", "features_per_example = collections.defaultdict(list)\n", "for i, feature in enumerate(features):\n", " features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)" ] }, { "cell_type": "markdown", "metadata": { "id": "gmDKzaD7vSzx" }, "source": [ "We're almost ready for our post-processing function. The last bit to deal with is the impossible answer (when `squad_v2 = True`). The code above only keeps answers that are inside the context, we need to also grab the score for the impossible answer (which has start and end indices corresponding to the index of the CLS token). When one example gives several features, we have to predict the impossible answer when all the features give a high score to the impossible answer (since one feature could predict the impossible answer just because the answer isn't in the part of the context it has access too), which is why the score of the impossible answer for one example is the *minimum* of the scores for the impossible answer in each feature generated by the example.\n", "\n", "We then predict the impossible answer when that score is greater than the score of the best non-impossible answer. All combined together, this gives us this post-processing function:" ] }, { "cell_type": "code", "execution_count": 55, "metadata": { "id": "_xVI5OkEvSzx" }, "outputs": [], "source": [ "from tqdm.auto import tqdm\n", "\n", "def postprocess_qa_predictions(examples, features, raw_predictions, n_best_size = 20, max_answer_length = 30):\n", " all_start_logits, all_end_logits = raw_predictions\n", " # Build a map example to its corresponding features.\n", " example_id_to_index = {k: i for i, k in enumerate(examples[\"id\"])}\n", " features_per_example = collections.defaultdict(list)\n", " for i, feature in enumerate(features):\n", " features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)\n", "\n", " # The dictionaries we have to fill.\n", " predictions = collections.OrderedDict()\n", "\n", " # Logging.\n", " print(f\"Post-processing {len(examples)} example predictions split into {len(features)} features.\")\n", "\n", " # Let's loop over all the examples!\n", " for example_index, example in enumerate(tqdm(examples)):\n", " # Those are the indices of the features associated to the current example.\n", " feature_indices = features_per_example[example_index]\n", "\n", " min_null_score = None # Only used if squad_v2 is True.\n", " valid_answers = []\n", " \n", " context = example[\"context\"]\n", " # Looping through all the features associated to the current example.\n", " for feature_index in feature_indices:\n", " # We grab the predictions of the model for this feature.\n", " start_logits = all_start_logits[feature_index]\n", " end_logits = all_end_logits[feature_index]\n", " # This is what will allow us to map some the positions in our logits to span of texts in the original\n", " # context.\n", " offset_mapping = features[feature_index][\"offset_mapping\"]\n", "\n", " # Update minimum null prediction.\n", " cls_index = features[feature_index][\"input_ids\"].index(tokenizer.cls_token_id)\n", " feature_null_score = start_logits[cls_index] + end_logits[cls_index]\n", " if min_null_score is None or min_null_score < feature_null_score:\n", " min_null_score = feature_null_score\n", "\n", " # Go through all possibilities for the `n_best_size` greater start and end logits.\n", " start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n", " end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n", " for start_index in start_indexes:\n", " for end_index in end_indexes:\n", " # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n", " # to part of the input_ids that are not in the context.\n", " if (\n", " start_index >= len(offset_mapping)\n", " or end_index >= len(offset_mapping)\n", " or offset_mapping[start_index] is None\n", " or offset_mapping[end_index] is None\n", " ):\n", " continue\n", " # Don't consider answers with a length that is either < 0 or > max_answer_length.\n", " if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n", " continue\n", "\n", " start_char = offset_mapping[start_index][0]\n", " end_char = offset_mapping[end_index][1]\n", " valid_answers.append(\n", " {\n", " \"score\": start_logits[start_index] + end_logits[end_index],\n", " \"text\": context[start_char: end_char]\n", " }\n", " )\n", " \n", " if len(valid_answers) > 0:\n", " best_answer = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[0]\n", " else:\n", " # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid\n", " # failure.\n", " best_answer = {\"text\": \"\", \"score\": 0.0}\n", " \n", " # Let's pick our final answer: the best one or the null answer (only for squad_v2)\n", " if not squad_v2:\n", " predictions[example[\"id\"]] = best_answer[\"text\"]\n", " else:\n", " answer = best_answer[\"text\"] if best_answer[\"score\"] > min_null_score else \"\"\n", " predictions[example[\"id\"]] = answer\n", "\n", " return predictions" ] }, { "cell_type": "markdown", "metadata": { "id": "foxGFk3cvSzy" }, "source": [ "And we can apply our post-processing function to our raw predictions:" ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "colab": { "referenced_widgets": [ "347ebed36d3541388e4e821372e91aa4" ] }, "id": "cF6upjjpvSzy", "outputId": "676b93ae-33c1-48a4-f0db-2952c52d0a3b" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Post-processing 10570 example predictions split into 10784 features.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "153b6f8fff6a4217b1ec1f001032c5e5", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/10570 [00:00:1: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library πŸ€— Evaluate: https://huggingface.co/docs/evaluate\n", " metric = load_metric(\"squad_v2\" if squad_v2 else \"squad\")\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1ec39dce486e4aad87b2284007e62ec2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Downloading builder script: 0%| | 0.00/1.72k [00:00