File size: 66,220 Bytes
821537b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 |
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.40.2)\n",
"Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.19.1)\n",
"Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
"Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.23.0)\n",
"Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.2)\n",
"Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
"Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.4.28)\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
"Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
"Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n",
"Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n",
"Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (16.0.0)\n",
"Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
"Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
"Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n",
"Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
"Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
"Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets) (2023.10.0)\n",
"Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.0b0)\n",
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.8.0)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.1.0)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n",
"Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
"Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
"Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
"Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"# Transformers installation\n",
"# ! pip install transformers datasets\n",
"# To install from source instead of the last release, comment the command above and uncomment the following one.\n",
"# ! pip install git+https://github.com/huggingface/transformers.git"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Causal language modeling"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"There are two types of language modeling, causal and masked. This guide illustrates causal language modeling.\n",
"Causal language models are frequently used for text generation. You can use these models for creative applications like\n",
"choosing your own text adventure or an intelligent coding assistant like Copilot or CodeParrot."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"cellView": "form",
"hide_input": true
},
"outputs": [],
"source": [
"# #@title\n",
"# from IPython.display import HTML\n",
"\n",
"# HTML('<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/Vpjb1lu0MDk?rel=0&controls=0&showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on\n",
"the left. This means the model cannot see future tokens. GPT-2 is an example of a causal language model.\n",
"\n",
"This guide will show you how to:\n",
"\n",
"1. Finetune [DistilGPT2](https://huggingface.co/distilgpt2) on the [r/askscience](https://www.reddit.com/r/askscience/) subset of the [ELI5](https://huggingface.co/datasets/eli5) dataset.\n",
"2. Use your finetuned model for inference.\n",
"\n",
"<Tip>\n",
"You can finetune other architectures for causal language modeling following the same steps in this guide.\n",
"Choose one of the following architectures:\n",
"\n",
"<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->\n",
"[BART](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/bart), [BERT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/bert), [Bert Generation](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/bert-generation), [BigBird](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/big_bird), [BigBird-Pegasus](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/bigbird_pegasus), [BioGpt](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/biogpt), [Blenderbot](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/blenderbot), [BlenderbotSmall](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/blenderbot-small), [BLOOM](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/bloom), [CamemBERT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/camembert), [CodeGen](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/codegen), [CPM-Ant](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/cpmant), [CTRL](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/ctrl), [Data2VecText](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/data2vec-text), [ELECTRA](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/electra), [ERNIE](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/ernie), [GIT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/git), [GPT-Sw3](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gpt-sw3), [OpenAI GPT-2](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gpt2), [GPTBigCode](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gpt_bigcode), [GPT Neo](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gpt_neo), [GPT NeoX](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gpt_neox), [GPT NeoX Japanese](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gpt_neox_japanese), [GPT-J](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/gptj), [LLaMA](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/llama), [Marian](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/marian), [mBART](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/mbart), [MEGA](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/mega), [Megatron-BERT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/megatron-bert), [MVP](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/mvp), [OpenLlama](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/open-llama), [OpenAI GPT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/openai-gpt), [OPT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/opt), [Pegasus](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/pegasus), [PLBart](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/plbart), [ProphetNet](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/prophetnet), [QDQBert](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/qdqbert), [Reformer](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/reformer), [RemBERT](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/rembert), [RoBERTa](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/roberta), [RoBERTa-PreLayerNorm](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/roberta-prelayernorm), [RoCBert](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/roc_bert), [RoFormer](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/roformer), [RWKV](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/rwkv), [Speech2Text2](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/speech_to_text_2), [Transformer-XL](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/transfo-xl), [TrOCR](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/trocr), [XGLM](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xglm), [XLM](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xlm), [XLM-ProphetNet](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xlm-prophetnet), [XLM-RoBERTa](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xlm-roberta), [XLM-RoBERTa-XL](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xlm-roberta-xl), [XLNet](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xlnet), [X-MOD](https://huggingface.co/docs/transformers/main/en/tasks/../model_doc/xmod)\n",
"\n",
"\n",
"<!--End of the generated tip-->\n",
"\n",
"</Tip>\n",
"\n",
"Before you begin, make sure you have all the necessary libraries installed:\n",
"\n",
"```bash\n",
"pip install transformers datasets evaluate\n",
"```\n",
"\n",
"We encourage you to log in to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to log in:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a6d9e280e08e40ddbbcb8fbe97e1fae9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# from huggingface_hub import notebook_login\n",
"\n",
"# notebook_login()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load ELI5 dataset"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Start by loading a smaller subset of the r/askscience subset of the ELI5 dataset from the 🤗 Datasets library.\n",
" This'll give you a chance to experiment and make sure everything works before spending more time training on the full dataset."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# from datasets import load_dataset\n",
"\n",
"# eli5 = load_dataset(\"eli5\", split=\"train_asks[:5000]\")"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e5c92a52c290468496943cb8023e4479",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating train split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "cf14d12614594f51b63d4aa8259d278f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating validation split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from datasets import load_dataset\n",
"# Falcon = load_dataset(\"csv\", data_files=\"FalconData.csv\")\n",
"Falcon = load_dataset('csv', data_files={\"train\": 'FalconData.csv', \"validation\": 'FalconDataEval.csv'})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Split the dataset's `train_asks` split into a train and test set with the [train_test_split](https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.train_test_split) method:"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# Falcon = Falcon.train_test_split(test_size=0.10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Then take a look at an example:"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'Text': 'Allow me to clarify a genuine fast for amateur online users What exactly is Youtube . com? Youtube . com is probably the most in-demand web site on the web which allow you to view and publish video lessons for free. These are generally submitted by Vimeo members on this video discussing system. Yet another thing YouTube registration is provided for free so anyone can join, however account is not required for watching video lessons. In order to sometimes observe video clips or post your own video lessons so that you can show to your friends, loved ones as well as other Vimeo members. Once you get dependent at viewing video clip, it is possible to phone yourself a YouTuber!\\n- Everything you are unable to upload? Nonetheless there are some regulations or YouTube\\'s regards to use that you should.\\n- Observing a Vimeo movie is really simple, you just need to.\\nObserving a You tube movie is absolutely simple, you just need to variety your best song or television set plan from the research discipline click on \"Research\" option and that\\'s it. It will approach your demand and give you a list of related results. You are able to click on a outcome and this will commence taking part in the recording. youtube downloader\\nAble to click on a outcome and\\nSo, just how to publish your chosen videos? Youtube . com is very popular online video discussing foundation that allows one to publish their video lessons. Uploading a relevant video online is an easy process, just select any video submit through your computer on your YouTube accounts webpage and it will surely begin posting the video. Nonetheless Vimeo will not offer any choice to down load a printed video that you will be seeing, you can easily take note of the site Link so that you can view it later, which seems handy for YouTube users.\\nEverything you cannot upload? Nevertheless there are a few regulations or YouTube\\'s terms of use that you have to comply with, specifically you happen to be unacceptable to upload any restricted content or erotic information. Nevertheless you can use it to showcase your products online.\\nA few regulations or\\nOnline video good quality once you upload Vimeo permits you to post all popular movie formats and produces good quality probable. Whenever you post a youtube video to Youtube . com, you ought to anticipate that high quality will slightly be changed, it is because YouTube optimizes the video for speedier packing. You can even add Hi-def or Hi-def video lessons nevertheless it will take much longer to weight once you observe it. Greater the high quality more slowly movie will load.\\nYou upload Vimeo\\nProbably the most well-known movie web sites online is You tube as well as for certain, you can find videos inside the web site you want to create you everywhere and adding it inside your PSP device might be what you need. However, YouTube video lessons will not be quickly down loadable. You might need a downloader to download the recording through the website and shop it inside your personal computer. video downloader\\nAfter you have saved the recording, it may possibly not certainly be around the preferred format which can be legible along with your Playstation portable. For those who have saved a structure not in mp4, you may want to transform the submit with your Computer in to a Playstation portable-pleasant structure. You may need a video clip converter for this task, and when you have changed the video tutorials, anyone can down load these to your Playstation portable.\\nWith your Playstation portable For those\\nIn accessing, simply link up your Playstation portable to the laptop or computer by means of its cord, use the Universal serial bus setting and download the video lessons and music that you want to bring along.\\nThat will help you look for a converter or a video downloader, specifically if you want to obtain video clips from Vimeo, be involved in forums and discover topics relevant to this. Certainly, you will also find a great deal of PSP movie information that may also assist you in making the best from your gadget and help you learn to see a number of videos on your gadget.\\nAlso find a great deal of PSP\\nYou can even get into membership web sites where PSP enthusiast collect and discuss information and facts and even more importantly, offers you the tools and software program that you will want to save music, videos and media records to your devices and permit you to enjoy the gizmo a lot more. Although these membership internet sites require only a minimum cost, it really is however vital that you are working with and creating dealings in a guaranteed and harmless internet site.\\n- You can even get into membership websites.\\n- One of the more preferred video clip sites on.\\n- Video quality when you post Vimeo enables you.\\n0 thoughts on “The Most Effective and Well-liked you tube downloader6675”'}"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Falcon['train'][0]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'Text': 'For some reason, removing motor grease from cotton-poly blend is perceived as one of the more difficult laundry problems out there. The truth is, that there are several methods that you can use to get rid of this type of stain, which are listed here. While some of these methods may seem a little strange, each and every one of them will work. All you need to do is be willing to try it. If you are hesitant about using any of these methods at all, be sure to test them out on a similar piece of fabric to see what the end result will be. If there is any damage to your particular piece of fabric, than do not use the method to happen to have a few white t-shirts, blouses, or button-up shirts, then chances are you know the pain of having to ...Discover More\\nTablecloths are not cheap, and it is always a great idea to protect anything that is expensive. Cleaning tablecloths is ...Discover More\\nWhile it can be annoying to find that your white apparel and linens have turned yellow in the laundry, it no longer needs ...Discover More\\nFREE SERVICE: Get tips like this every week in Cleaning Tips from Tips.Net. Enter your address and click \"Subscribe.\"\\nView most recent newsletter.\\n2015-08-29 08:54:35\\nJune\\nComing from a long line of mechanics, I\\'ve always kept a bottle of LESTOIL around...works GREAT on auto grease, and cooking grease as well, just follow the directions on the bottle.\\nFREE SERVICE: Get tips like this every week in Cleaning Tips from Tips.Net. Enter your address and click \"Subscribe.\"\\n(Your e-mail address is not shared with anyone, ever.)\\nView the most recent newsletter.'}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Falcon['validation'][0]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"While this may look like a lot, you're only really interested in the `text` field. What's cool about language modeling\n",
"tasks is you don't need labels (also known as an unsupervised task) because the next word *is* the label."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Preprocess"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"hide_input": true
},
"outputs": [
{
"data": {
"text/html": [
"<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/ma1TrR7gE7I?rel=0&controls=0&showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# #@title\n",
"# from IPython.display import HTML\n",
"\n",
"# HTML('<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/ma1TrR7gE7I?rel=0&controls=0&showinfo=0\" frameborder=\"0\" allowfullscreen></iframe>')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The next step is to load a DistilGPT2 tokenizer to process the `text` subfield:"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
" warnings.warn(\n",
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"from transformers import AutoTokenizer, GPT2TokenizerFast\n",
"\n",
"# tokenizer = AutoTokenizer.from_pretrained(\"distilgpt2\")\n",
"\n",
"\n",
"tokenizer = GPT2TokenizerFast.from_pretrained(\"Xenova/gpt-4\")#, cache_dir=cache_dir)\n",
"tokenizer.pad_token = tokenizer.eos_token"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You'll notice from the example above, the `text` field is actually nested inside `answers`. This means you'll need to\n",
"extract the `text` subfield from its nested structure with the [`flatten`](https://huggingface.co/docs/datasets/process.html#flatten) method:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'Text': 'Allow me to clarify a genuine fast for amateur online users What exactly is Youtube . com? Youtube . com is probably the most in-demand web site on the web which allow you to view and publish video lessons for free. These are generally submitted by Vimeo members on this video discussing system. Yet another thing YouTube registration is provided for free so anyone can join, however account is not required for watching video lessons. In order to sometimes observe video clips or post your own video lessons so that you can show to your friends, loved ones as well as other Vimeo members. Once you get dependent at viewing video clip, it is possible to phone yourself a YouTuber!\\n- Everything you are unable to upload? Nonetheless there are some regulations or YouTube\\'s regards to use that you should.\\n- Observing a Vimeo movie is really simple, you just need to.\\nObserving a You tube movie is absolutely simple, you just need to variety your best song or television set plan from the research discipline click on \"Research\" option and that\\'s it. It will approach your demand and give you a list of related results. You are able to click on a outcome and this will commence taking part in the recording. youtube downloader\\nAble to click on a outcome and\\nSo, just how to publish your chosen videos? Youtube . com is very popular online video discussing foundation that allows one to publish their video lessons. Uploading a relevant video online is an easy process, just select any video submit through your computer on your YouTube accounts webpage and it will surely begin posting the video. Nonetheless Vimeo will not offer any choice to down load a printed video that you will be seeing, you can easily take note of the site Link so that you can view it later, which seems handy for YouTube users.\\nEverything you cannot upload? Nevertheless there are a few regulations or YouTube\\'s terms of use that you have to comply with, specifically you happen to be unacceptable to upload any restricted content or erotic information. Nevertheless you can use it to showcase your products online.\\nA few regulations or\\nOnline video good quality once you upload Vimeo permits you to post all popular movie formats and produces good quality probable. Whenever you post a youtube video to Youtube . com, you ought to anticipate that high quality will slightly be changed, it is because YouTube optimizes the video for speedier packing. You can even add Hi-def or Hi-def video lessons nevertheless it will take much longer to weight once you observe it. Greater the high quality more slowly movie will load.\\nYou upload Vimeo\\nProbably the most well-known movie web sites online is You tube as well as for certain, you can find videos inside the web site you want to create you everywhere and adding it inside your PSP device might be what you need. However, YouTube video lessons will not be quickly down loadable. You might need a downloader to download the recording through the website and shop it inside your personal computer. video downloader\\nAfter you have saved the recording, it may possibly not certainly be around the preferred format which can be legible along with your Playstation portable. For those who have saved a structure not in mp4, you may want to transform the submit with your Computer in to a Playstation portable-pleasant structure. You may need a video clip converter for this task, and when you have changed the video tutorials, anyone can down load these to your Playstation portable.\\nWith your Playstation portable For those\\nIn accessing, simply link up your Playstation portable to the laptop or computer by means of its cord, use the Universal serial bus setting and download the video lessons and music that you want to bring along.\\nThat will help you look for a converter or a video downloader, specifically if you want to obtain video clips from Vimeo, be involved in forums and discover topics relevant to this. Certainly, you will also find a great deal of PSP movie information that may also assist you in making the best from your gadget and help you learn to see a number of videos on your gadget.\\nAlso find a great deal of PSP\\nYou can even get into membership web sites where PSP enthusiast collect and discuss information and facts and even more importantly, offers you the tools and software program that you will want to save music, videos and media records to your devices and permit you to enjoy the gizmo a lot more. Although these membership internet sites require only a minimum cost, it really is however vital that you are working with and creating dealings in a guaranteed and harmless internet site.\\n- You can even get into membership websites.\\n- One of the more preferred video clip sites on.\\n- Video quality when you post Vimeo enables you.\\n0 thoughts on “The Most Effective and Well-liked you tube downloader6675”'}"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Falcon = Falcon.flatten()\n",
"Falcon[\"train\"][0]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Each subfield is now a separate column as indicated by the `answers` prefix, and the `text` field is a list now. Instead\n",
"of tokenizing each sentence separately, convert the list to a string so you can jointly tokenize them.\n",
"\n",
"Here is a first preprocessing function to join the list of strings for each example and tokenize the result:"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def preprocess_function(examples):\n",
" return tokenizer([\" \".join(x) for x in examples[\"Text\"]])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To apply this preprocessing function over the entire dataset, use the 🤗 Datasets [map](https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map) method. You can speed up the `map` function by setting `batched=True` to process multiple elements of the dataset at once, and increasing the number of processes with `num_proc`. Remove any columns you don't need:"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The OrderedVocab you are attempting to save contains holes for indices [100256, 100261, 100262, 100263, 100266, 100267, 100268, 100269, 100270, 100271, 100272, 100273, 100274, 100275], your vocabulary could be corrupted !\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "51bff46d94664c468064b17d1a8bf1c0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Map (num_proc=4): 0%| | 0/20000 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Token indices sequence length is longer than the specified maximum sequence length for this model (8569 > 8192). Running this sequence through the model will result in indexing errors\n",
"Token indices sequence length is longer than the specified maximum sequence length for this model (14224 > 8192). Running this sequence through the model will result in indexing errors\n",
"Token indices sequence length is longer than the specified maximum sequence length for this model (15104 > 8192). Running this sequence through the model will result in indexing errors\n",
"Token indices sequence length is longer than the specified maximum sequence length for this model (32874 > 8192). Running this sequence through the model will result in indexing errors\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"The OrderedVocab you are attempting to save contains holes for indices [100256, 100261, 100262, 100263, 100266, 100267, 100268, 100269, 100270, 100271, 100272, 100273, 100274, 100275], your vocabulary could be corrupted !\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5a093fd9868042a9ac76ed1c141711a7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Map (num_proc=4): 0%| | 0/2000 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Token indices sequence length is longer than the specified maximum sequence length for this model (8414 > 8192). Running this sequence through the model will result in indexing errors\n",
"Token indices sequence length is longer than the specified maximum sequence length for this model (11892 > 8192). Running this sequence through the model will result in indexing errors\n",
"Token indices sequence length is longer than the specified maximum sequence length for this model (22303 > 8192). Running this sequence through the model will result in indexing errors\n",
"Token indices sequence length is longer than the specified maximum sequence length for this model (12749 > 8192). Running this sequence through the model will result in indexing errors\n"
]
}
],
"source": [
"tokenized_Falcon = Falcon.map(\n",
" preprocess_function,\n",
" batched=True,\n",
" num_proc=4,\n",
" remove_columns=Falcon[\"train\"].column_names,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This dataset contains the token sequences, but some of these are longer than the maximum input length for the model.\n",
"\n",
"You can now use a second preprocessing function to\n",
"- concatenate all the sequences\n",
"- split the concatenated sequences into shorter chunks defined by `block_size`, which should be both shorter than the maximum input length and short enough for your GPU RAM."
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"block_size = 1048\n",
"\n",
"\n",
"def group_texts(examples):\n",
" # Concatenate all texts.\n",
" concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
" total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
" # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
" # customize this part to your needs.\n",
" if total_length >= block_size:\n",
" total_length = (total_length // block_size) * block_size\n",
" # Split by chunks of block_size.\n",
" result = {\n",
" k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n",
" for k, t in concatenated_examples.items()\n",
" }\n",
" result[\"labels\"] = result[\"input_ids\"].copy()\n",
" return result"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Apply the `group_texts` function over the entire dataset:"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6134c09493054ce3940da711dc2e965e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Map (num_proc=4): 0%| | 0/20000 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "bd3f26e9c76f42f1827aa11aa45416df",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Map (num_proc=4): 0%| | 0/2000 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"lm_dataset = tokenized_Falcon.map(group_texts, batched=True, num_proc=4)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now create a batch of examples using [DataCollatorForLanguageModeling](https://huggingface.co/docs/transformers/main/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling). It's more efficient to *dynamically pad* the\n",
"sentences to the longest length in a batch during collation, instead of padding the whole dataset to the maximum length.\n",
"\n",
"Use the end-of-sequence token as the padding token and set `mlm=False`. This will use the inputs as labels shifted to the right by one element:"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from transformers import DataCollatorForLanguageModeling\n",
"\n",
"tokenizer.pad_token = tokenizer.eos_token\n",
"data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<Tip>\n",
"\n",
"If you aren't familiar with finetuning a model with the [Trainer](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer), take a look at the [basic tutorial](https://huggingface.co/docs/transformers/main/en/tasks/../training#train-with-pytorch-trainer)!\n",
"\n",
"</Tip>\n",
"\n",
"You're ready to start training your model now! Load DistilGPT2 with [AutoModelForCausalLM](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoModelForCausalLM):"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
" warnings.warn(\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f55ae69743a74a08943641e2da03e791",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from transformers import AutoModelForCausalLM, TrainingArguments, Trainer\n",
"import torch\n",
"model = AutoModelForCausalLM.from_pretrained(\"tensorplex-labs/pretraining-sn9-7B-5\", torch_dtype=torch.bfloat16) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"At this point, only three steps remain:\n",
"\n",
"1. Define your training hyperparameters in [TrainingArguments](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments). The only required parameter is `output_dir` which specifies where to save your model. You'll push this model to the Hub by setting `push_to_hub=True` (you need to be signed in to Hugging Face to upload your model).\n",
"2. Pass the training arguments to [Trainer](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer) along with the model, datasets, and data collator.\n",
"3. Call [train()](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.train) to finetune your model."
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"torch.cuda.empty_cache()\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import gc\n",
"\n",
"# del tensor_name # Delete the tensor\n",
"gc.collect() # Collect garbage\n",
"torch.cuda.empty_cache() # Clear cache"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"torch.cuda.empty_cache()"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<torch.autograd.grad_mode.no_grad at 0x7f41880db6d0>"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"torch.no_grad()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"LlamaForCausalLM(\n",
" (model): LlamaModel(\n",
" (embed_tokens): Embedding(100288, 4096)\n",
" (layers): ModuleList(\n",
" (0-29): 30 x LlamaDecoderLayer(\n",
" (self_attn): LlamaSdpaAttention(\n",
" (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
" (k_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
" (v_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
" (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
" (rotary_emb): LlamaRotaryEmbedding()\n",
" )\n",
" (mlp): LlamaMLP(\n",
" (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
" (up_proj): Linear(in_features=4096, out_features=11008, bias=False)\n",
" (down_proj): Linear(in_features=11008, out_features=4096, bias=False)\n",
" (act_fn): SiLU()\n",
" )\n",
" (input_layernorm): LlamaRMSNorm()\n",
" (post_attention_layernorm): LlamaRMSNorm()\n",
" )\n",
" )\n",
" (norm): LlamaRMSNorm()\n",
" )\n",
" (lm_head): Linear(in_features=4096, out_features=100288, bias=False)\n",
")"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.to('cuda')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"training_args = TrainingArguments(\n",
" output_dir=\"Fine-Tuned-S9\",\n",
" bf16=True,\n",
" # evaluation_strategy=\"epoch\",\n",
" evaluation_strategy=\"steps\",\n",
" learning_rate=2e-5,\n",
" weight_decay=0.01,\n",
" num_train_epochs=1,\n",
" per_device_train_batch_size=2,\n",
" per_device_eval_batch_size=2,\n",
" # lr_scheduler_type = 'cosine',\n",
" push_to_hub=False,\n",
" save_total_limit = 2\n",
" # save_strategy = “no”\n",
" load_best_model_at_end=False\n",
")\n",
"\n",
"trainer = Trainer(\n",
" model=model,\n",
" args=training_args,\n",
" train_dataset=lm_dataset[\"train\"],\n",
" eval_dataset=lm_dataset[\"validation\"],\n",
" # eval_dataset=lm_dataset[\"test\"],\n",
" data_collator=data_collator,\n",
")\n",
"\n",
"# trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"trainer.train()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Once training is completed, use the [evaluate()](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.evaluate) method to evaluate your model and get its perplexity:"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Perplexity: 2.21\n"
]
}
],
"source": [
"import math\n",
"\n",
"eval_results = trainer.evaluate()\n",
"print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Then share your model to the Hub with the [push_to_hub()](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.push_to_hub) method so everyone can use your model:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# trainer.push_to_hub()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<Tip>\n",
"\n",
"For a more in-depth example of how to finetune a model for causal language modeling, take a look at the corresponding\n",
"[PyTorch notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb)\n",
"or [TensorFlow notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb).\n",
"\n",
"</Tip>"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Inference"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Great, now that you've finetuned a model, you can use it for inference!\n",
"\n",
"Come up with a prompt you'd like to generate text from:"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# prompt = \"Somatic hypermutation allows the immune system to\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The simplest way to try out your finetuned model for inference is to use it in a [pipeline()](https://huggingface.co/docs/transformers/main/en/main_classes/pipelines#transformers.pipeline). Instantiate a `pipeline` for text generation with your model, and pass your text to it:"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Could not load model Fine-Tuned-S9/checkpoint-4000 with any of the following classes: (<class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'>, <class 'transformers.models.llama.modeling_llama.LlamaForCausalLM'>). See the original errors:\n\nwhile loading with AutoModelForCausalLM, an error is thrown:\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\", line 283, in infer_framework_load_model\n model = model_class.from_pretrained(model, **kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py\", line 563, in from_pretrained\n return model_class.from_pretrained(\n File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 3260, in from_pretrained\n raise EnvironmentError(\nOSError: Error no file named pytorch_model.bin, tf_model.h5, model.ckpt.index or flax_model.msgpack found in directory Fine-Tuned-S9/checkpoint-4000.\n\nwhile loading with LlamaForCausalLM, an error is thrown:\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\", line 283, in infer_framework_load_model\n model = model_class.from_pretrained(model, **kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 3260, in from_pretrained\n raise EnvironmentError(\nOSError: Error no file named pytorch_model.bin, tf_model.h5, model.ckpt.index or flax_model.msgpack found in directory Fine-Tuned-S9/checkpoint-4000.\n\n\n",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[20], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtransformers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m pipeline\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# checkpoint-4000\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m generator \u001b[38;5;241m=\u001b[39m \u001b[43mpipeline\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext-generation\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mFine-Tuned-S9/checkpoint-4000\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m generator(prompt)\n",
"File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/__init__.py:906\u001b[0m, in \u001b[0;36mpipeline\u001b[0;34m(task, model, config, tokenizer, feature_extractor, image_processor, framework, revision, use_fast, token, device, device_map, torch_dtype, trust_remote_code, model_kwargs, pipeline_class, **kwargs)\u001b[0m\n\u001b[1;32m 904\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(model, \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m framework \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 905\u001b[0m model_classes \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtf\u001b[39m\u001b[38;5;124m\"\u001b[39m: targeted_task[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtf\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m: targeted_task[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m]}\n\u001b[0;32m--> 906\u001b[0m framework, model \u001b[38;5;241m=\u001b[39m \u001b[43minfer_framework_load_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 907\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 908\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_classes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_classes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 909\u001b[0m \u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 910\u001b[0m \u001b[43m \u001b[49m\u001b[43mframework\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mframework\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 911\u001b[0m \u001b[43m \u001b[49m\u001b[43mtask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 912\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhub_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 913\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 914\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 916\u001b[0m model_config \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mconfig\n\u001b[1;32m 917\u001b[0m hub_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_commit_hash\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39m_commit_hash\n",
"File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:296\u001b[0m, in \u001b[0;36minfer_framework_load_model\u001b[0;34m(model, config, model_classes, task, framework, **model_kwargs)\u001b[0m\n\u001b[1;32m 294\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m class_name, trace \u001b[38;5;129;01min\u001b[39;00m all_traceback\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 295\u001b[0m error \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwhile loading with \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mclass_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, an error is thrown:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mtrace\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 296\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 297\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCould not load model \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m with any of the following classes: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mclass_tuple\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. See the original errors:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00merror\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 298\u001b[0m )\n\u001b[1;32m 300\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m framework \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 301\u001b[0m framework \u001b[38;5;241m=\u001b[39m infer_framework(model\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n",
"\u001b[0;31mValueError\u001b[0m: Could not load model Fine-Tuned-S9/checkpoint-4000 with any of the following classes: (<class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'>, <class 'transformers.models.llama.modeling_llama.LlamaForCausalLM'>). See the original errors:\n\nwhile loading with AutoModelForCausalLM, an error is thrown:\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\", line 283, in infer_framework_load_model\n model = model_class.from_pretrained(model, **kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py\", line 563, in from_pretrained\n return model_class.from_pretrained(\n File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 3260, in from_pretrained\n raise EnvironmentError(\nOSError: Error no file named pytorch_model.bin, tf_model.h5, model.ckpt.index or flax_model.msgpack found in directory Fine-Tuned-S9/checkpoint-4000.\n\nwhile loading with LlamaForCausalLM, an error is thrown:\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py\", line 283, in infer_framework_load_model\n model = model_class.from_pretrained(model, **kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 3260, in from_pretrained\n raise EnvironmentError(\nOSError: Error no file named pytorch_model.bin, tf_model.h5, model.ckpt.index or flax_model.msgpack found in directory Fine-Tuned-S9/checkpoint-4000.\n\n\n"
]
}
],
"source": [
"# from transformers import pipeline\n",
"# # checkpoint-4000\n",
"# generator = pipeline(\"text-generation\", model=\"Fine-Tuned-S9/checkpoint-4000\")\n",
"# generator(prompt)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Tokenize the text and return the `input_ids` as PyTorch tensors:"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"# from transformers import AutoTokenizer\n",
"\n",
"# tokenizer = AutoTokenizer.from_pretrained(\"Xenova/gpt-4\")\n",
"# inputs = tokenizer(prompt, return_tensors=\"pt\").input_ids"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Use the [generate()](https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate) method to generate text.\n",
"For more details about the different text generation strategies and parameters for controlling generation, check out the [Text generation strategies](https://huggingface.co/docs/transformers/main/en/tasks/../generation_strategies) page."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7ba147780e8548d28a00a655e81e588a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"config.json: 0%| | 0.00/688 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
" warnings.warn(\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "04e2f536d4d1492bbb4dcf72abbf2cc3",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model.safetensors.index.json: 0%| | 0.00/22.5k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "df7e14c799c0457f8422442a065f3b03",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading shards: 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ee74102a34694e6cb57a00210d34cf19",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00001-of-00003.safetensors: 0%| | 0.00/4.97G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "978d214714044affb97e1b31ab6deafc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00002-of-00003.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0a2fb5b3f2ec4e3e8d7bc9db54a0635e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00003-of-00003.safetensors: 0%| | 0.00/3.84G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Error while downloading from https://cdn-lfs-us-1.huggingface.co/repos/54/cf/54cf63a091d3be4443d28131b5c3686f6dd17bc8fe13dfd74b30bc4eafc5b3e2/4c4148f267d0c0cb2979c9cf8e60f11fb91770076c28a2a79f4446ea30bff523?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00003-of-00003.safetensors%3B+filename%3D%22model-00003-of-00003.safetensors%22%3B&Expires=1715867899&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNTg2Nzg5OX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzU0L2NmLzU0Y2Y2M2EwOTFkM2JlNDQ0M2QyODEzMWI1YzM2ODZmNmRkMTdiYzhmZTEzZGZkNzRiMzBiYzRlYWZjNWIzZTIvNGM0MTQ4ZjI2N2QwYzBjYjI5NzljOWNmOGU2MGYxMWZiOTE3NzAwNzZjMjhhMmE3OWY0NDQ2ZWEzMGJmZjUyMz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=NRnXWL-gncnyNfcEhT0Xqi7WNbx5rVxELBfBIjnfb3zk7DCNDIqSPi-iNcrXmNkEmINWGbghFy4ifzUqvzNOmm0cJF10hMi%7E6R5DBKRBK0DRGtC2fC72sXzk9ysyJ6mQRSegUeDZy2KZqUL3wzwRC2Xhv8baK%7ENi0FGjUSh0Hmpg7Wgbs2quZRMM7lXqI-y3bkKh7L6OBXnx3W55Mlzzt87CgYLyotXuFIUrQ1W5lN6R3LWZuDvJ0ClLVuSKjTGwBv9MRQYLewybb4yqSmmEDfTkmuCphg2%7EfzNJ53Q2kqMEVC6gRPf67v8NDR9j57zOtoNSc1-SdaCem95aycbC7A__&Key-Pair-Id=KCD77M1F0VK2B: HTTPSConnectionPool(host='cdn-lfs-us-1.huggingface.co', port=443): Read timed out.\n",
"Trying to resume download...\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "635db10feaa74dff93285752d9e79520",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00003-of-00003.safetensors: 71%|####### | 2.71G/3.84G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "38e479e6424d4edc8d00795ce084d4c2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "602b879326a44c58bc0909a3b86cd666",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"generation_config.json: 0%| | 0.00/121 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
"Setting `pad_token_id` to `eos_token_id`:100257 for open-end generation.\n"
]
}
],
"source": [
"# from transformers import AutoModelForCausalLM\n",
"\n",
"# model = AutoModelForCausalLM.from_pretrained(\"deepnet/SN6-BestLlama\")\n",
"# outputs = model.generate(inputs, max_new_tokens=100, do_sample=True, top_k=50, top_p=0.95)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Decode the generated token ids back into text:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Somatic hypermutation allows the immune system to recognize foreign proteins. \\n - . \\n - \\n 1 . 3 \\n S e t s \\n 0 \\n A c c e p t s \\n A l m o s t \\n 1 \\n C l o s e d \\n T o p i c s \\n P a p e r s \\n 0 \\n P a p e r s \\n B e a r i n g \\n P a g e s \\n 0 \\n P a g e s \\n R e c o']"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# tokenizer.batch_decode(outputs, skip_special_tokens=True)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Somatic hypermutation allows the immune system to recognize foreign proteins. \\n - . \\n - \\n 1 . 3 \\n S e t s \\n 0 \\n A c c e p t s \\n A l m o s t \\n 1 \\n C l o s e d \\n T o p i c s \\n P a p e r s \\n 0 \\n P a p e r s \\n B e a r i n g \\n P a g e s \\n 0 \\n P a g e s \\n R e c o']"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# tokenizer.batch_decode(outputs, skip_special_tokens=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
|