Upload EXL2_Private_Quant_V2.ipynb

EXL2_Private_Quant_V2.ipynb

@@ -0,0 +1,191 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Quantizing huggingface models to exl2\n",
+        "This version of my exl2 quantize colab creates a single quantizaion to upload privatly.\\\n",
+        "To calculate an estimate for VRAM size use: [NyxKrage/LLM-Model-VRAM-Calculator](https://huggingface.co/spaces/NyxKrage/LLM-Model-VRAM-Calculator)\\\n",
+        "Not all models and architectures are compatible with exl2."
+      ],
+      "metadata": {
+        "id": "Ku0ezvyD42ng"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "G7zSk2LWHtPU"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Download and install environment\n",
+        "!git clone https://github.com/turboderp/exllamav2\n",
+        "%cd exllamav2\n",
+        "print(\"Installing pip dependencies\")\n",
+        "!pip install -q -r requirements.txt\n",
+        "!pip install -q huggingface_hub requests tqdm\n",
+        "#@markdown Uses [download-model.py](https://github.com/oobabooga/text-generation-webui/blob/main/download-model.py) by [oobabooga](https://github.com/oobabooga)\n",
+        "!wget https://raw.githubusercontent.com/oobabooga/text-generation-webui/main/download-model.py\n",
+        "model = \"none\"\n",
+        "dsd = 'false'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title Login to HF (Required to upload files)\n",
+        "#@markdown From my Colab/Kaggle login script on [Anthonyg5005/hf-scripts](https://huggingface.co/Anthonyg5005/hf-scripts/blob/main/HF%20Login%20Snippet%20Kaggle.py)\n",
+        "#import required functions\n",
+        "import os\n",
+        "from huggingface_hub import login, get_token, whoami\n",
+        "\n",
+        "#get token\n",
+        "if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', None) is not None: #check if user in kaggle\n",
+        "    from kaggle_secrets import UserSecretsClient\n",
+        "    from kaggle_web_client import BackendError\n",
+        "    try:\n",
+        "        login(UserSecretsClient().get_secret(\"HF_TOKEN\")) #login if token secret found\n",
+        "    except BackendError:\n",
+        "        print('''\n",
+        "            When using Kaggle, make sure to use the secret key HF_TOKEN with a 'WRITE' token.\n",
+        "                   This will prevent the need to login every time you run the script.\n",
+        "                   Set your secrets with the secrets add-on on the top of the screen.\n",
+        "             ''')\n",
+        "if get_token() is not None:\n",
+        "    #if the token is found then log in:\n",
+        "    login(get_token())\n",
+        "else:\n",
+        "    #if the token is not found then prompt user to provide it:\n",
+        "    login(input(\"API token not detected. Enter your HuggingFace (WRITE) token: \"))\n",
+        "\n",
+        "#if the token is read only then prompt user to provide a write token (Only required if user needs a WRITE token, remove if READ is enough):\n",
+        "while True:\n",
+        "    if whoami().get('auth', {}).get('accessToken', {}).get('role', None) != 'write':\n",
+        "        if os.environ.get('HF_TOKEN', None) is not None: #if environ finds HF_TOKEN as read-only then display following text and exit:\n",
+        "            print('''\n",
+        "          You have the environment variable HF_TOKEN set.\n",
+        "          You cannot log in.\n",
+        "          Either set the environment variable to a 'WRITE' token or remove it.\n",
+        "                  ''')\n",
+        "            input(\"Press enter to continue.\")\n",
+        "            exit()\n",
+        "        if os.environ.get('COLAB_BACKEND_VERSION', None) is not None:\n",
+        "            print('''\n",
+        "                              Your Colab secret key is read-only\n",
+        "                Please switch your key to 'write' or disable notebook access on the left.\n",
+        "                               For now, you are stuck in a loop\n",
+        "                  ''')\n",
+        "        elif os.environ.get('KAGGLE_KERNEL_RUN_TYPE', None) is not None:\n",
+        "            print('''\n",
+        "                                      Your Kaggle secret key is read-only\n",
+        "                Please switch your key to 'write' or unattach from notebook in add-ons at the top.\n",
+        "                          Having a read-only key attched will require login every time.\n",
+        "                ''')\n",
+        "        print(\"You do not have write access to this repository. Please use a valid token with (WRITE) access.\")\n",
+        "        login(input(\"Enter your HuggingFace (WRITE) token: \"))\n",
+        "        continue\n",
+        "    break"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "8Hl3fQmRLybp"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title ##Choose HF model to download\n",
+        "#@markdown ###Repo should be formatted as user/repo\n",
+        "#@markdown Weights must be stored in safetensors\n",
+        "if model != \"none\":\n",
+        "    !rm {model}-{BPW}bpw.zip\n",
+        "    !rm -r {model}-exl2-{BPW}bpw\n",
+        "repo_url = \"mistralai/Mistral-7B-Instruct-v0.2\" # @param {type:\"string\"}\n",
+        "model = repo_url.replace(\"/\", \"_\")\n",
+        "!python download-model.py {repo_url}"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "NI1LUMD7H-Zx"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title Quantize the model\n",
+        "#@markdown ###Takes ~13 minutes to start quantizing first time, then quantization will last based on model size\n",
+        "#@markdown Target bits per weight:\n",
+        "BPW = \"4.125\" # @param {type:\"string\"}\n",
+        "!mkdir {model}-exl2-{BPW}bpw-WD\n",
+        "!mkdir {model}-exl2-{BPW}bpw\n",
+        "!cp models/{model}/config.json {model}-exl2-{BPW}bpw-WD\n",
+        "#@markdown Calibrate with dataset, may improve model output (optional):\n",
+        "Calibrate = True # @param {type:\"boolean\"}\n",
+        "#@markdown Calibration dataset, enable calibrate above (must be parquet file):\n",
+        "if Calibrate == True:\n",
+        "    dataset_url = \"https://huggingface.co/datasets/wikitext/resolve/refs%2Fconvert%2Fparquet/wikitext-103-v1/test/0000.parquet?download=true\" # @param {type:\"string\"}\n",
+        "    dataset_url = dataset_url.replace(\"?download=true\", \"\")\n",
+        "    if dsd == 'false':\n",
+        "        !wget {dataset_url}\n",
+        "        dsd = 'true'\n",
+        "    dataset = dataset_url.split(\"/\")[-1]\n",
+        "#@markdown To use a calibration dataset, enter the huggingface resolve url. Right click the download button and copy the link. Afterwards, paste the link into dataset_url.\n",
+        "#@markdown ![Example Image](https://huggingface.co/Anthonyg5005/hf-scripts/resolve/main/Screenshot%202024-03-17%20011855.png \"Copy from download button\")\n",
+        "if Calibrate == True:\n",
+        "    quant = f\"convert.py -i models/{model} -o {model}-exl2-{BPW}bpw-WD -cf {model}-exl2-{BPW}bpw -c {dataset} -b {BPW}\"\n",
+        "else:\n",
+        "    quant = f\"convert.py -i models/{model} -o {model}-exl2-{BPW}bpw-WD -cf {model}-exl2-{BPW}bpw -b {BPW}\"\n",
+        "!python {quant}"
+      ],
+      "metadata": {
+        "id": "8anbEbGyNmBI",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title Upload to huggingface privately\n",
+        "#@markdown You may also set it to public but I'd recommend waiting for my next ipynb that will create mutliple quants and place them all into individual branches.\n",
+        "!rm -r {model}-exl2-{BPW}bpw-WD\n",
+        "!rm -r models/{model}\n",
+        "print(\"Uploading to Huggingface. May take a while\")\n",
+        "from huggingface_hub import HfApi, whoami, create_repo\n",
+        "create_repo(f\"{whoami().get('name', None)}/{model}-exl2-{BPW}bpw\", private=True)\n",
+        "HfApi().upload_folder(folder_path=f\"{model}-exl2-{BPW}bpw\", repo_id=f\"{whoami().get('name', None)}/{model}-exl2-{BPW}bpw\", repo_type=\"model\", commit_message=\"Upload from Colab automation\")\n",
+        "print(f\"uploaded to {whoami().get('name', None)}/{model}-exl2-{BPW}bpw\")"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "XORLS2uPrbma"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}