bf16 to fp16

Untitled.ipynb

@@ -0,0 +1,175 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6363d4dd-5c18-4652-b682-bf54a468aae5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from safetensors import safe_open\n",
+    "from safetensors.torch import save_file\n",
+    "\n",
+    "# 加载 safetensors 文件\n",
+    "model_path = \"model.safetensors\"\n",
+    "with safe_open(model_path, framework=\"pt\", device=\"cpu\") as f:\n",
+    "    state_dict = {key: f.get_tensor(key) for key in f.keys()}\n",
+    "\n",
+    "# 将 BF16 转换为 FP16\n",
+    "fp16_state_dict = {key: value.to(torch.float16) for key, value in state_dict.items()}\n",
+    "\n",
+    "# 保存为新的 safetensors 文件\n",
+    "output_path = \"model_fp16.safetensors\"\n",
+    "save_file(fp16_state_dict, output_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9aa60eaa-883a-465e-958a-3280e0439f66",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "PackageNotFoundError",
+     "evalue": "No package metadata was found for auto-gptq",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:397\u001b[0m, in \u001b[0;36mDistribution.from_name\u001b[0;34m(cls, name)\u001b[0m\n\u001b[1;32m    396\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 397\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdiscover\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    398\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n",
+      "\u001b[0;31mStopIteration\u001b[0m: ",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[0;31mPackageNotFoundError\u001b[0m                      Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[4], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m model_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# 加载模型，显式指定 torch_dtype\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mAutoModelForCausalLM\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtorch_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat16\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# 或 torch.float16，与权重一致\u001b[39;49;00m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdevice_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mauto\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# 可选，自动分配设备\u001b[39;49;00m\n\u001b[1;32m     11\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;66;03m# 加载分词器\u001b[39;00m\n\u001b[1;32m     14\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m AutoTokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(model_path)\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:564\u001b[0m, in \u001b[0;36m_BaseAutoModelClass.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    562\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(config) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m    563\u001b[0m     model_class \u001b[38;5;241m=\u001b[39m _get_model_class(config, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping)\n\u001b[0;32m--> 564\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    565\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhub_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m    566\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    567\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    568\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnrecognized configuration class \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconfig\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m for this kind of AutoModel: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    569\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mModel type should be one of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(c\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39mc\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    570\u001b[0m )\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/site-packages/transformers/modeling_utils.py:3669\u001b[0m, in \u001b[0;36mPreTrainedModel.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   3666\u001b[0m     hf_quantizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   3668\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hf_quantizer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 3669\u001b[0m     \u001b[43mhf_quantizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_environment\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtorch_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch_dtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3671\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfrom_tf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_tf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3672\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfrom_flax\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_flax\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3673\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdevice_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice_map\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3674\u001b[0m \u001b[43m        \u001b[49m\u001b[43mweights_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mweights_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3675\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3676\u001b[0m     torch_dtype \u001b[38;5;241m=\u001b[39m hf_quantizer\u001b[38;5;241m.\u001b[39mupdate_torch_dtype(torch_dtype)\n\u001b[1;32m   3677\u001b[0m     device_map \u001b[38;5;241m=\u001b[39m hf_quantizer\u001b[38;5;241m.\u001b[39mupdate_device_map(device_map)\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/site-packages/transformers/quantizers/quantizer_gptq.py:52\u001b[0m, in \u001b[0;36mGptqHfQuantizer.validate_environment\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mvalidate_environment\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 52\u001b[0m     gptq_supports_cpu \u001b[38;5;241m=\u001b[39m version\u001b[38;5;241m.\u001b[39mparse(\u001b[43mimportlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mversion\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mauto-gptq\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m) \u001b[38;5;241m>\u001b[39m version\u001b[38;5;241m.\u001b[39mparse(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0.4.2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     53\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m gptq_supports_cpu \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available():\n\u001b[1;32m     54\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGPU is required to quantize or run quantize model.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:889\u001b[0m, in \u001b[0;36mversion\u001b[0;34m(distribution_name)\u001b[0m\n\u001b[1;32m    882\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mversion\u001b[39m(distribution_name):\n\u001b[1;32m    883\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Get the version string for the named package.\u001b[39;00m\n\u001b[1;32m    884\u001b[0m \n\u001b[1;32m    885\u001b[0m \u001b[38;5;124;03m    :param distribution_name: The name of the distribution package to query.\u001b[39;00m\n\u001b[1;32m    886\u001b[0m \u001b[38;5;124;03m    :return: The version string for the package as defined in the package's\u001b[39;00m\n\u001b[1;32m    887\u001b[0m \u001b[38;5;124;03m        \"Version\" metadata key.\u001b[39;00m\n\u001b[1;32m    888\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 889\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdistribution\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdistribution_name\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mversion\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:862\u001b[0m, in \u001b[0;36mdistribution\u001b[0;34m(distribution_name)\u001b[0m\n\u001b[1;32m    856\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdistribution\u001b[39m(distribution_name):\n\u001b[1;32m    857\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Get the ``Distribution`` instance for the named package.\u001b[39;00m\n\u001b[1;32m    858\u001b[0m \n\u001b[1;32m    859\u001b[0m \u001b[38;5;124;03m    :param distribution_name: The name of the distribution package as a string.\u001b[39;00m\n\u001b[1;32m    860\u001b[0m \u001b[38;5;124;03m    :return: A ``Distribution`` instance (or subclass thereof).\u001b[39;00m\n\u001b[1;32m    861\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 862\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mDistribution\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_name\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdistribution_name\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:399\u001b[0m, in \u001b[0;36mDistribution.from_name\u001b[0;34m(cls, name)\u001b[0m\n\u001b[1;32m    397\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mdiscover(name\u001b[38;5;241m=\u001b[39mname))\n\u001b[1;32m    398\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[0;32m--> 399\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m PackageNotFoundError(name)\n",
+      "\u001b[0;31mPackageNotFoundError\u001b[0m: No package metadata was found for auto-gptq"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "# 模型路径\n",
+    "model_path = \"./\"\n",
+    "\n",
+    "# 加载模型，显式指定 torch_dtype\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    model_path,\n",
+    "    torch_dtype=torch.float16,  # 或 torch.float16，与权重一致\n",
+    "    device_map=\"auto\"  # 可选，自动分配设备\n",
+    ")\n",
+    "\n",
+    "# 加载分词器\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
+    "\n",
+    "# 测试模型\n",
+    "input_text = \"Hello, how are you?\"\n",
+    "inputs = tokenizer(input_text, return_tensors=\"pt\")\n",
+    "outputs = model.generate(**inputs)\n",
+    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "9d15b55d-f5bc-40ad-80e5-81d421bd0592",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Looking in indexes: http://mirrors.aliyun.com/pypi/simple\n",
+      "Collecting auto-gptq\n",
+      "  Downloading http://mirrors.aliyun.com/pypi/packages/90/e5/b22697903982284fe284568fb2663a2196694a8eee637f5cf4ccfe435a38/auto_gptq-0.7.1.tar.gz (126 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.1/126.1 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25hDiscarding \u001b[4;34mhttp://mirrors.aliyun.com/pypi/packages/90/e5/b22697903982284fe284568fb2663a2196694a8eee637f5cf4ccfe435a38/auto_gptq-0.7.1.tar.gz#sha256=5c61ad380e9b4c603757c254765e9083a90a820cd0aff1b5d2c6f7fd96c85e80 (from http://mirrors.aliyun.com/pypi/simple/auto-gptq/) (requires-python:>=3.8.0)\u001b[0m: \u001b[33mRequested auto-gptq from http://mirrors.aliyun.com/pypi/packages/90/e5/b22697903982284fe284568fb2663a2196694a8eee637f5cf4ccfe435a38/auto_gptq-0.7.1.tar.gz#sha256=5c61ad380e9b4c603757c254765e9083a90a820cd0aff1b5d2c6f7fd96c85e80 has inconsistent version: expected '0.7.1', but metadata has '0.7.1+cu1241'\u001b[0m\n",
+      "  Downloading http://mirrors.aliyun.com/pypi/packages/34/71/c3e73cf17681f6ff4754ef8f4cb8b67af3def230fc8711eac1250bbd78d5/auto_gptq-0.7.0.tar.gz (124 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.6/124.6 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25hDiscarding \u001b[4;34mhttp://mirrors.aliyun.com/pypi/packages/34/71/c3e73cf17681f6ff4754ef8f4cb8b67af3def230fc8711eac1250bbd78d5/auto_gptq-0.7.0.tar.gz#sha256=50a5396fae2db5a19446b3198ef0e86ee520846b881db47bdbf4eb9260eac723 (from http://mirrors.aliyun.com/pypi/simple/auto-gptq/) (requires-python:>=3.8.0)\u001b[0m: \u001b[33mRequested auto-gptq from http://mirrors.aliyun.com/pypi/packages/34/71/c3e73cf17681f6ff4754ef8f4cb8b67af3def230fc8711eac1250bbd78d5/auto_gptq-0.7.0.tar.gz#sha256=50a5396fae2db5a19446b3198ef0e86ee520846b881db47bdbf4eb9260eac723 has inconsistent version: expected '0.7.0', but metadata has '0.7.0+cu1241'\u001b[0m\n",
+      "  Downloading http://mirrors.aliyun.com/pypi/packages/49/af/02b66e55dfd9aeb0ece923843043724ed7432ec0c649ea0f3b9fa1dd90c6/auto_gptq-0.6.0.tar.gz (120 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.0/121.0 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25lerror\n",
+      "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
+      "  \n",
+      "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
+      "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
+      "  \u001b[31m╰─>\u001b[0m \u001b[31m[20 lines of output]\u001b[0m\n",
+      "  \u001b[31m   \u001b[0m python: can't open file '/tmp/pip-install-9tzda5pv/auto-gptq_60a9cac9f2ef4bb89bb1ce3f0d0126b0/./autogptq_extension/qigen/generate.py': [Errno 2] No such file or directory\n",
+      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
+      "  \u001b[31m   \u001b[0m   File \"/tmp/pip-install-9tzda5pv/auto-gptq_60a9cac9f2ef4bb89bb1ce3f0d0126b0/setup.py\", line 109, in <module>\n",
+      "  \u001b[31m   \u001b[0m     subprocess.check_output([\"python\", \"./autogptq_extension/qigen/generate.py\", \"--module\", \"--search\", \"--p\", str(p)])\n",
+      "  \u001b[31m   \u001b[0m   File \"/root/miniconda3/lib/python3.12/subprocess.py\", line 466, in check_output\n",
+      "  \u001b[31m   \u001b[0m     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,\n",
+      "  \u001b[31m   \u001b[0m            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "  \u001b[31m   \u001b[0m   File \"/root/miniconda3/lib/python3.12/subprocess.py\", line 571, in run\n",
+      "  \u001b[31m   \u001b[0m     raise CalledProcessError(retcode, process.args,\n",
+      "  \u001b[31m   \u001b[0m subprocess.CalledProcessError: Command '['python', './autogptq_extension/qigen/generate.py', '--module', '--search', '--p', '48']' returned non-zero exit status 2.\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m During handling of the above exception, another exception occurred:\n",
+      "  \u001b[31m   \u001b[0m \n",
+      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
+      "  \u001b[31m   \u001b[0m   File \"<string>\", line 2, in <module>\n",
+      "  \u001b[31m   \u001b[0m   File \"<pip-setuptools-caller>\", line 34, in <module>\n",
+      "  \u001b[31m   \u001b[0m   File \"/tmp/pip-install-9tzda5pv/auto-gptq_60a9cac9f2ef4bb89bb1ce3f0d0126b0/setup.py\", line 111, in <module>\n",
+      "  \u001b[31m   \u001b[0m     raise Exception(f\"Generating QiGen kernels failed with the error shown above.\")\n",
+      "  \u001b[31m   \u001b[0m Exception: Generating QiGen kernels failed with the error shown above.\n",
+      "  \u001b[31m   \u001b[0m Generating qigen kernels...\n",
+      "  \u001b[31m   \u001b[0m \u001b[31m[end of output]\u001b[0m\n",
+      "  \n",
+      "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
+      "\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
+      "\n",
+      "\u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
+      "\u001b[31m╰─>\u001b[0m See above for output.\n",
+      "\n",
+      "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
+      "\u001b[1;36mhint\u001b[0m: See above for details.\n",
+      "\u001b[?25h"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install auto-gptq"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b8c14f99-2926-433f-8249-58cc123ce73d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

config.json

@@ -1,6 +1,6 @@
 {
   "_attn_implementation_autoset": true,
-  "_name_or_path": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775",
+  "_name_or_path": "Qwen2.5-0.5B-Instruct-FP16",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
@@ -43,7 +43,7 @@
   "rope_theta": 1000000.0,
   "sliding_window": null,
   "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.47.1",
   "use_cache": true,
   "use_sliding_window": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8572721112376195fbcc8595a6a204b3bfad41f90101ba5d4e36d205c718e79f
-size 641496752
+oid sha256:50b54dd93397a6b301fa3c735b5d2763fbcacc9bf63e7cb76e9159a7b378c3ee
+size 641496696

model_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03dc9e241100a8782ad4c8d5c4751bbaf8c02b8a0360f02f84dcb5973925bb79
+size 595928496

quant_log.csv

@@ -1,169 +1,169 @@
 layer,module,loss,damp,time
-0,self_attn.k_proj,0.00138,0.01000,0.372
-0,self_attn.v_proj,0.00003,0.01000,0.143
-0,self_attn.q_proj,0.01032,0.01000,0.146
-0,self_attn.o_proj,0.00001,0.01000,0.141
-0,mlp.up_proj,0.09639,0.01000,0.147
-0,mlp.gate_proj,0.19389,0.01000,0.144
-0,mlp.down_proj,0.00145,0.01000,0.760
-1,self_attn.k_proj,0.00983,0.01000,0.141
-1,self_attn.v_proj,0.00101,0.01000,0.140
-1,self_attn.q_proj,0.03416,0.01000,0.145
-1,self_attn.o_proj,0.00206,0.01000,0.142
-1,mlp.up_proj,0.10123,0.01000,0.152
-1,mlp.gate_proj,0.19488,0.01000,0.147
-1,mlp.down_proj,0.00114,0.01000,0.763
-2,self_attn.k_proj,0.01729,0.01000,0.143
-2,self_attn.v_proj,0.00211,0.01000,0.141
-2,self_attn.q_proj,0.05901,0.01000,0.143
-2,self_attn.o_proj,0.00084,0.01000,0.144
-2,mlp.up_proj,0.16170,0.01000,0.144
-2,mlp.gate_proj,0.30168,0.01000,0.144
-2,mlp.down_proj,3.40212,0.01000,0.777
-3,self_attn.k_proj,0.02164,0.01000,0.146
-3,self_attn.v_proj,0.00550,0.01000,0.144
-3,self_attn.q_proj,0.08517,0.01000,0.140
-3,self_attn.o_proj,0.00069,0.01000,0.145
-3,mlp.up_proj,1.55705,0.01000,0.149
-3,mlp.gate_proj,14.22241,0.01000,0.153
-3,mlp.down_proj,10.47623,0.01000,0.771
-4,self_attn.k_proj,0.01608,0.01000,0.141
-4,self_attn.v_proj,0.00733,0.01000,0.140
-4,self_attn.q_proj,0.07782,0.01000,0.141
-4,self_attn.o_proj,0.00357,0.01000,0.142
-4,mlp.up_proj,0.26308,0.01000,0.146
-4,mlp.gate_proj,0.44777,0.01000,0.146
-4,mlp.down_proj,0.00393,0.01000,0.773
-5,self_attn.k_proj,0.01548,0.01000,0.141
-5,self_attn.v_proj,0.00733,0.01000,0.143
-5,self_attn.q_proj,0.07514,0.01000,0.142
-5,self_attn.o_proj,0.00126,0.01000,0.142
-5,mlp.up_proj,0.32005,0.01000,0.147
-5,mlp.gate_proj,0.75867,0.01000,0.143
-5,mlp.down_proj,0.00588,0.01000,0.782
-6,self_attn.k_proj,0.02100,0.01000,0.142
-6,self_attn.v_proj,0.00581,0.01000,0.142
-6,self_attn.q_proj,0.08125,0.01000,0.143
-6,self_attn.o_proj,0.00086,0.01000,0.142
-6,mlp.up_proj,0.25930,0.01000,0.146
-6,mlp.gate_proj,0.43217,0.01000,0.147
-6,mlp.down_proj,0.00290,0.01000,0.776
-7,self_attn.k_proj,0.02048,0.01000,0.140
-7,self_attn.v_proj,0.00687,0.01000,0.139
-7,self_attn.q_proj,0.08387,0.01000,0.140
-7,self_attn.o_proj,0.00102,0.01000,0.143
-7,mlp.up_proj,0.27253,0.01000,0.144
-7,mlp.gate_proj,0.38177,0.01000,0.150
-7,mlp.down_proj,0.00328,0.01000,0.786
-8,self_attn.k_proj,0.01920,0.01000,0.144
-8,self_attn.v_proj,0.00666,0.01000,0.139
-8,self_attn.q_proj,0.08629,0.01000,0.142
-8,self_attn.o_proj,0.00223,0.01000,0.145
-8,mlp.up_proj,0.24209,0.01000,0.145
-8,mlp.gate_proj,0.34371,0.01000,0.147
-8,mlp.down_proj,0.00234,0.01000,0.770
-9,self_attn.k_proj,0.03560,0.01000,0.146
-9,self_attn.v_proj,0.01327,0.01000,0.146
-9,self_attn.q_proj,0.15881,0.01000,0.142
-9,self_attn.o_proj,0.00107,0.01000,0.141
-9,mlp.up_proj,0.24913,0.01000,0.145
-9,mlp.gate_proj,0.31356,0.01000,0.153
-9,mlp.down_proj,0.00273,0.01000,0.770
-10,self_attn.k_proj,0.01898,0.01000,0.140
-10,self_attn.v_proj,0.01226,0.01000,0.142
-10,self_attn.q_proj,0.09338,0.01000,0.146
-10,self_attn.o_proj,0.00256,0.01000,0.139
-10,mlp.up_proj,0.19837,0.01000,0.146
-10,mlp.gate_proj,0.25931,0.01000,0.152
-10,mlp.down_proj,0.00200,0.01000,0.781
-11,self_attn.k_proj,0.03752,0.01000,0.142
-11,self_attn.v_proj,0.01387,0.01000,0.139
-11,self_attn.q_proj,0.17363,0.01000,0.144
-11,self_attn.o_proj,0.00194,0.01000,0.142
-11,mlp.up_proj,0.24216,0.01000,0.147
-11,mlp.gate_proj,0.26251,0.01000,0.147
-11,mlp.down_proj,0.00267,0.01000,0.776
-12,self_attn.k_proj,0.01698,0.01000,0.142
-12,self_attn.v_proj,0.00979,0.01000,0.142
-12,self_attn.q_proj,0.08870,0.01000,0.143
-12,self_attn.o_proj,0.00231,0.01000,0.141
-12,mlp.up_proj,0.20748,0.01000,0.146
-12,mlp.gate_proj,0.22905,0.01000,0.148
-12,mlp.down_proj,0.00239,0.01000,0.772
-13,self_attn.k_proj,0.02764,0.01000,0.143
-13,self_attn.v_proj,0.01191,0.01000,0.145
-13,self_attn.q_proj,0.14133,0.01000,0.146
-13,self_attn.o_proj,0.00121,0.01000,0.144
-13,mlp.up_proj,0.24341,0.01000,0.149
-13,mlp.gate_proj,0.24755,0.01000,0.147
-13,mlp.down_proj,0.00313,0.01000,0.776
-14,self_attn.k_proj,0.02157,0.01000,0.144
-14,self_attn.v_proj,0.01694,0.01000,0.143
-14,self_attn.q_proj,0.12651,0.01000,0.145
-14,self_attn.o_proj,0.00391,0.01000,0.143
-14,mlp.up_proj,0.23182,0.01000,0.145
-14,mlp.gate_proj,0.26455,0.01000,0.157
-14,mlp.down_proj,0.00381,0.01000,0.783
-15,self_attn.k_proj,0.01991,0.01000,0.145
-15,self_attn.v_proj,0.01372,0.01000,0.144
-15,self_attn.q_proj,0.10473,0.01000,0.146
-15,self_attn.o_proj,0.00366,0.01000,0.145
-15,mlp.up_proj,0.29403,0.01000,0.149
-15,mlp.gate_proj,0.30643,0.01000,0.145
-15,mlp.down_proj,0.00572,0.01000,0.765
-16,self_attn.k_proj,0.03065,0.01000,0.144
-16,self_attn.v_proj,0.02840,0.01000,0.140
-16,self_attn.q_proj,0.21804,0.01000,0.146
-16,self_attn.o_proj,0.00136,0.01000,0.143
-16,mlp.up_proj,0.34904,0.01000,0.146
-16,mlp.gate_proj,0.45293,0.01000,0.147
-16,mlp.down_proj,0.01072,0.01000,0.766
-17,self_attn.k_proj,0.02077,0.01000,0.145
-17,self_attn.v_proj,0.02172,0.01000,0.145
-17,self_attn.q_proj,0.13563,0.01000,0.144
-17,self_attn.o_proj,0.00379,0.01000,0.142
-17,mlp.up_proj,0.40654,0.01000,0.150
-17,mlp.gate_proj,0.63069,0.01000,0.153
-17,mlp.down_proj,0.00786,0.01000,0.771
-18,self_attn.k_proj,0.02198,0.01000,0.149
-18,self_attn.v_proj,0.01774,0.01000,0.141
-18,self_attn.q_proj,0.10674,0.01000,0.145
-18,self_attn.o_proj,0.00129,0.01000,0.143
-18,mlp.up_proj,0.38269,0.01000,0.148
-18,mlp.gate_proj,0.47625,0.01000,0.147
-18,mlp.down_proj,0.00765,0.01000,0.777
-19,self_attn.k_proj,0.01776,0.01000,0.144
-19,self_attn.v_proj,0.01706,0.01000,0.142
-19,self_attn.q_proj,0.11138,0.01000,0.143
-19,self_attn.o_proj,0.00345,0.01000,0.143
-19,mlp.up_proj,0.57152,0.01000,0.146
-19,mlp.gate_proj,0.68538,0.01000,0.148
-19,mlp.down_proj,0.01574,0.01000,0.783
-20,self_attn.k_proj,0.02558,0.01000,0.145
-20,self_attn.v_proj,0.05435,0.01000,0.146
-20,self_attn.q_proj,0.16357,0.01000,0.140
-20,self_attn.o_proj,0.01051,0.01000,0.144
-20,mlp.up_proj,0.65037,0.01000,0.145
-20,mlp.gate_proj,0.71838,0.01000,0.147
-20,mlp.down_proj,0.02858,0.01000,0.766
-21,self_attn.k_proj,0.03181,0.01000,0.144
-21,self_attn.v_proj,0.10603,0.01000,0.144
-21,self_attn.q_proj,0.23383,0.01000,0.144
-21,self_attn.o_proj,0.02307,0.01000,0.144
-21,mlp.up_proj,1.75373,0.01000,0.158
-21,mlp.gate_proj,3.94375,0.01000,0.146
-21,mlp.down_proj,17.75035,0.01000,0.779
-22,self_attn.k_proj,0.02056,0.01000,0.141
-22,self_attn.v_proj,0.06087,0.01000,0.140
-22,self_attn.q_proj,0.15773,0.01000,0.148
-22,self_attn.o_proj,0.01726,0.01000,0.140
-22,mlp.up_proj,0.53556,0.01000,0.144
-22,mlp.gate_proj,0.51474,0.01000,0.144
-22,mlp.down_proj,0.04907,0.01000,0.769
-23,self_attn.k_proj,0.02615,0.01000,0.176
-23,self_attn.v_proj,0.05774,0.01000,0.209
-23,self_attn.q_proj,0.20516,0.01000,0.209
-23,self_attn.o_proj,0.03206,0.01000,0.210
-23,mlp.up_proj,0.83435,0.01000,0.214
-23,mlp.gate_proj,0.87210,0.01000,0.224
-23,mlp.down_proj,0.48671,0.01000,0.784
+0,self_attn.k_proj,0.00136,0.01000,0.325
+0,self_attn.v_proj,0.00003,0.01000,0.129
+0,self_attn.q_proj,0.01018,0.01000,0.137
+0,self_attn.o_proj,0.00001,0.01000,0.134
+0,mlp.up_proj,0.09437,0.01000,0.139
+0,mlp.gate_proj,0.18865,0.01000,0.139
+0,mlp.down_proj,0.00143,0.01000,0.735
+1,self_attn.k_proj,0.00999,0.01000,0.135
+1,self_attn.v_proj,0.00103,0.01000,0.135
+1,self_attn.q_proj,0.03432,0.01000,0.135
+1,self_attn.o_proj,0.00249,0.01000,0.132
+1,mlp.up_proj,0.10145,0.01000,0.145
+1,mlp.gate_proj,0.19494,0.01000,0.138
+1,mlp.down_proj,0.00111,0.01000,0.741
+2,self_attn.k_proj,0.01739,0.01000,0.135
+2,self_attn.v_proj,0.00208,0.01000,0.130
+2,self_attn.q_proj,0.05895,0.01000,0.135
+2,self_attn.o_proj,0.00084,0.01000,0.132
+2,mlp.up_proj,0.15928,0.01000,0.139
+2,mlp.gate_proj,0.29904,0.01000,0.137
+2,mlp.down_proj,3.25813,0.01000,0.724
+3,self_attn.k_proj,0.02043,0.01000,0.141
+3,self_attn.v_proj,0.00490,0.01000,0.138
+3,self_attn.q_proj,0.08232,0.01000,0.139
+3,self_attn.o_proj,0.00070,0.01000,0.141
+3,mlp.up_proj,1.52620,0.01000,0.140
+3,mlp.gate_proj,13.87913,0.01000,0.153
+3,mlp.down_proj,3.20606,0.01000,0.734
+4,self_attn.k_proj,0.01640,0.01000,0.135
+4,self_attn.v_proj,0.00691,0.01000,0.131
+4,self_attn.q_proj,0.07569,0.01000,0.132
+4,self_attn.o_proj,0.00309,0.01000,0.134
+4,mlp.up_proj,0.24986,0.01000,0.136
+4,mlp.gate_proj,0.42210,0.01000,0.138
+4,mlp.down_proj,0.00362,0.01000,0.737
+5,self_attn.k_proj,0.01574,0.01000,0.137
+5,self_attn.v_proj,0.00762,0.01000,0.133
+5,self_attn.q_proj,0.07645,0.01000,0.136
+5,self_attn.o_proj,0.00106,0.01000,0.138
+5,mlp.up_proj,0.30390,0.01000,0.143
+5,mlp.gate_proj,0.71931,0.01000,0.147
+5,mlp.down_proj,0.00614,0.01000,0.733
+6,self_attn.k_proj,0.02019,0.01000,0.139
+6,self_attn.v_proj,0.00567,0.01000,0.138
+6,self_attn.q_proj,0.07846,0.01000,0.140
+6,self_attn.o_proj,0.00102,0.01000,0.140
+6,mlp.up_proj,0.25922,0.01000,0.147
+6,mlp.gate_proj,0.42813,0.01000,0.144
+6,mlp.down_proj,0.00318,0.01000,0.753
+7,self_attn.k_proj,0.02236,0.01000,0.144
+7,self_attn.v_proj,0.00721,0.01000,0.146
+7,self_attn.q_proj,0.08888,0.01000,0.135
+7,self_attn.o_proj,0.00106,0.01000,0.140
+7,mlp.up_proj,0.27111,0.01000,0.147
+7,mlp.gate_proj,0.37811,0.01000,0.143
+7,mlp.down_proj,0.00381,0.01000,0.762
+8,self_attn.k_proj,0.01918,0.01000,0.137
+8,self_attn.v_proj,0.00650,0.01000,0.138
+8,self_attn.q_proj,0.08556,0.01000,0.140
+8,self_attn.o_proj,0.00224,0.01000,0.140
+8,mlp.up_proj,0.23711,0.01000,0.140
+8,mlp.gate_proj,0.33877,0.01000,0.141
+8,mlp.down_proj,0.00247,0.01000,0.745
+9,self_attn.k_proj,0.03478,0.01000,0.141
+9,self_attn.v_proj,0.01390,0.01000,0.145
+9,self_attn.q_proj,0.16694,0.01000,0.139
+9,self_attn.o_proj,0.00133,0.01000,0.140
+9,mlp.up_proj,0.25157,0.01000,0.142
+9,mlp.gate_proj,0.31524,0.01000,0.148
+9,mlp.down_proj,0.00306,0.01000,0.780
+10,self_attn.k_proj,0.02065,0.01000,0.140
+10,self_attn.v_proj,0.01247,0.01000,0.139
+10,self_attn.q_proj,0.09851,0.01000,0.141
+10,self_attn.o_proj,0.00224,0.01000,0.141
+10,mlp.up_proj,0.20222,0.01000,0.145
+10,mlp.gate_proj,0.26411,0.01000,0.144
+10,mlp.down_proj,0.00195,0.01000,0.758
+11,self_attn.k_proj,0.03751,0.01000,0.138
+11,self_attn.v_proj,0.01373,0.01000,0.137
+11,self_attn.q_proj,0.16694,0.01000,0.138
+11,self_attn.o_proj,0.00163,0.01000,0.136
+11,mlp.up_proj,0.23553,0.01000,0.142
+11,mlp.gate_proj,0.25575,0.01000,0.141
+11,mlp.down_proj,0.00258,0.01000,0.766
+12,self_attn.k_proj,0.01699,0.01000,0.141
+12,self_attn.v_proj,0.00987,0.01000,0.143
+12,self_attn.q_proj,0.09044,0.01000,0.149
+12,self_attn.o_proj,0.00252,0.01000,0.145
+12,mlp.up_proj,0.20484,0.01000,0.146
+12,mlp.gate_proj,0.22553,0.01000,0.146
+12,mlp.down_proj,0.00257,0.01000,0.767
+13,self_attn.k_proj,0.02551,0.01000,0.135
+13,self_attn.v_proj,0.01122,0.01000,0.136
+13,self_attn.q_proj,0.13072,0.01000,0.146
+13,self_attn.o_proj,0.00144,0.01000,0.134
+13,mlp.up_proj,0.23847,0.01000,0.147
+13,mlp.gate_proj,0.24350,0.01000,0.151
+13,mlp.down_proj,0.00315,0.01000,0.778
+14,self_attn.k_proj,0.02141,0.01000,0.139
+14,self_attn.v_proj,0.01697,0.01000,0.135
+14,self_attn.q_proj,0.12928,0.01000,0.140
+14,self_attn.o_proj,0.00471,0.01000,0.136
+14,mlp.up_proj,0.22936,0.01000,0.139
+14,mlp.gate_proj,0.26044,0.01000,0.147
+14,mlp.down_proj,0.00375,0.01000,0.742
+15,self_attn.k_proj,0.02037,0.01000,0.137
+15,self_attn.v_proj,0.01484,0.01000,0.136
+15,self_attn.q_proj,0.10820,0.01000,0.140
+15,self_attn.o_proj,0.00345,0.01000,0.138
+15,mlp.up_proj,0.30909,0.01000,0.143
+15,mlp.gate_proj,0.32287,0.01000,0.149
+15,mlp.down_proj,0.00640,0.01000,0.748
+16,self_attn.k_proj,0.02956,0.01000,0.139
+16,self_attn.v_proj,0.02730,0.01000,0.136
+16,self_attn.q_proj,0.21341,0.01000,0.137
+16,self_attn.o_proj,0.00164,0.01000,0.142
+16,mlp.up_proj,0.36086,0.01000,0.146
+16,mlp.gate_proj,0.47303,0.01000,0.153
+16,mlp.down_proj,0.01073,0.01000,0.753
+17,self_attn.k_proj,0.02010,0.01000,0.144
+17,self_attn.v_proj,0.02237,0.01000,0.138
+17,self_attn.q_proj,0.13404,0.01000,0.138
+17,self_attn.o_proj,0.00350,0.01000,0.138
+17,mlp.up_proj,0.42924,0.01000,0.140
+17,mlp.gate_proj,0.66441,0.01000,0.142
+17,mlp.down_proj,0.00841,0.01000,0.736
+18,self_attn.k_proj,0.02365,0.01000,0.142
+18,self_attn.v_proj,0.01843,0.01000,0.133
+18,self_attn.q_proj,0.11482,0.01000,0.138
+18,self_attn.o_proj,0.00114,0.01000,0.136
+18,mlp.up_proj,0.41826,0.01000,0.142
+18,mlp.gate_proj,0.52589,0.01000,0.144
+18,mlp.down_proj,0.00837,0.01000,0.747
+19,self_attn.k_proj,0.01885,0.01000,0.144
+19,self_attn.v_proj,0.01739,0.01000,0.145
+19,self_attn.q_proj,0.11650,0.01000,0.139
+19,self_attn.o_proj,0.00337,0.01000,0.143
+19,mlp.up_proj,0.61293,0.01000,0.146
+19,mlp.gate_proj,0.73786,0.01000,0.147
+19,mlp.down_proj,0.01766,0.01000,0.743
+20,self_attn.k_proj,0.02636,0.01000,0.137
+20,self_attn.v_proj,0.05430,0.01000,0.137
+20,self_attn.q_proj,0.16743,0.01000,0.145
+20,self_attn.o_proj,0.01072,0.01000,0.140
+20,mlp.up_proj,0.71327,0.01000,0.136
+20,mlp.gate_proj,0.78980,0.01000,0.147
+20,mlp.down_proj,0.02940,0.01000,0.730
+21,self_attn.k_proj,0.03263,0.01000,0.137
+21,self_attn.v_proj,0.10656,0.01000,0.136
+21,self_attn.q_proj,0.22666,0.01000,0.139
+21,self_attn.o_proj,0.02572,0.01000,0.138
+21,mlp.up_proj,1.93951,0.01000,0.140
+21,mlp.gate_proj,4.54990,0.01000,0.143
+21,mlp.down_proj,17.82846,0.01000,0.752
+22,self_attn.k_proj,0.02069,0.01000,0.143
+22,self_attn.v_proj,0.06015,0.01000,0.136
+22,self_attn.q_proj,0.15573,0.01000,0.143
+22,self_attn.o_proj,0.01185,0.01000,0.138
+22,mlp.up_proj,0.59293,0.01000,0.144
+22,mlp.gate_proj,0.57112,0.01000,0.149
+22,mlp.down_proj,0.05477,0.01000,0.768
+23,self_attn.k_proj,0.02633,0.01000,0.137
+23,self_attn.v_proj,0.05556,0.01000,0.141
+23,self_attn.q_proj,0.19868,0.01000,0.136
+23,self_attn.o_proj,0.03036,0.01000,0.139
+23,mlp.up_proj,0.86650,0.01000,0.141
+23,mlp.gate_proj,0.89305,0.01000,0.142
+23,mlp.down_proj,0.48127,0.01000,0.736

untitled.py

@@ -0,0 +1,15 @@
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+# 加载 safetensors 文件
+model_path = "model.safetensors"
+with safe_open(model_path, framework="pt", device="cpu") as f:
+    state_dict = {key: f.get_tensor(key) for key in f.keys()}
+
+# 将 BF16 转换为 FP16
+fp16_state_dict = {key: value.to(torch.float16) for key, value in state_dict.items()}
+
+# 保存为新的 safetensors 文件
+output_path = "model_fp16.safetensors"
+save_file(fp16_state_dict, output_path)