Upload 2 files

hf_llm_metaeval_eval_harness_mixtral_mmlu.ipynb

@@ -12,31 +12,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "106470b4-1be2-4790-a5bf-485a1e3bb2d3",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Token has not been saved to git credential helper.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
-      "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
-      "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
-      "\n",
-      "git config --global credential.helper store\n",
-      "\n",
-      "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from datetime import datetime\n",
     "import os\n",
@@ -121,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "367d4e84-ba2a-42fb-a5a9-b303ab63dea5",
    "metadata": {},
    "outputs": [],
@@ -146,29 +125,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "2bc08732-dea6-43a4-8d31-c407069a6eef",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "No files have been modified since last commit. Skipping to prevent empty commit.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "hf_upload_folder(BASE_FOLDER)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "61157ee0-4cf8-473c-a2ff-f695890a738d",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

hf_llm_metaeval_eval_harness_mixtral_pub.ipynb

@@ -12,31 +12,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "106470b4-1be2-4790-a5bf-485a1e3bb2d3",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Token has not been saved to git credential helper.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
-      "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
-      "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
-      "\n",
-      "git config --global credential.helper store\n",
-      "\n",
-      "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from datetime import datetime\n",
     "import os\n",
@@ -76,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "4855d529-22d7-4eaf-b6bd-1cc40b0223a6",
    "metadata": {},
    "outputs": [],
@@ -137,110 +116,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "cf3725e2-3a26-49b7-a5ba-71ecb5ba28bc",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
-      "\t`--num_processes` was set to a value of `0`\n",
-      "\t`--num_machines` was set to a value of `1`\n",
-      "\t`--mixed_precision` was set to a value of `'no'`\n",
-      "\t`--dynamo_backend` was set to a value of `'no'`\n",
-      "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
-      "2024-12-07:13:34:28,398 INFO     [__main__.py:279] Verbosity set to INFO\n",
-      "2024-12-07:13:34:28,398 INFO     [__main__.py:303] Including path: ./pub/tasks/.\n",
-      "2024-12-07:13:34:38,122 INFO     [__main__.py:376] Selected Tasks: ['pub_task_1']\n",
-      "2024-12-07:13:34:38,124 INFO     [evaluator.py:164] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234\n",
-      "2024-12-07:13:34:38,124 INFO     [evaluator.py:201] Initializing hf model, with arguments: {'pretrained': 'mistralai/Mixtral-8x7B-Instruct-v0.1', 'revision': '41bd4c9e7e4fb318ca40e721131d4933966c2cc1', 'trust_remote_code': False, 'dtype': 'bfloat16', 'parallelize': True}\n",
-      "config.json: 100%|██████████████████████████████| 720/720 [00:00<00:00, 140kB/s]\n",
-      "tokenizer_config.json: 100%|████████████████| 2.10k/2.10k [00:00<00:00, 521kB/s]\n",
-      "tokenizer.model: 100%|████████████████████████| 493k/493k [00:00<00:00, 127MB/s]\n",
-      "tokenizer.json: 100%|██████████████████████| 1.80M/1.80M [00:00<00:00, 47.8MB/s]\n",
-      "special_tokens_map.json: 100%|██████████████████| 414/414 [00:00<00:00, 419kB/s]\n",
-      "2024-12-07:13:34:38,668 INFO     [huggingface.py:351] Model parallel was set to True, setting max memory per GPU to {} and device map to auto\n",
-      "model.safetensors.index.json: 100%|████████| 92.7k/92.7k [00:00<00:00, 17.7MB/s]\n",
-      "Downloading shards:   0%|                                | 0/19 [00:00<?, ?it/s]\n",
-      "model-00001-of-00019.safetensors:   0%|             | 0.00/4.89G [00:00<?, ?B/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   0%|    | 21.0M/4.89G [00:00<01:59, 40.9MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   1%|    | 31.5M/4.89G [00:00<01:41, 47.8MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   1%|    | 41.9M/4.89G [00:00<01:39, 48.9MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   1%|    | 52.4M/4.89G [00:01<01:43, 46.6MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   2%|    | 73.4M/4.89G [00:01<01:43, 46.8MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   2%|    | 83.9M/4.89G [00:01<01:34, 51.0MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   2%|     | 105M/4.89G [00:02<02:01, 39.4MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   2%|     | 115M/4.89G [00:02<02:12, 36.1MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   3%|▏    | 136M/4.89G [00:03<02:07, 37.3MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   3%|▏    | 147M/4.89G [00:03<01:51, 42.7MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   3%|▏    | 168M/4.89G [00:03<01:38, 48.1MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   4%|▏    | 178M/4.89G [00:04<01:38, 47.9MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   4%|▏    | 199M/4.89G [00:04<01:36, 48.5MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   4%|▏    | 210M/4.89G [00:04<01:39, 47.1MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   5%|▏    | 231M/4.89G [00:05<01:36, 48.3MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   5%|▏    | 241M/4.89G [00:05<01:34, 49.2MB/s]\u001b[A\n",
-      "model-00001-of-00019.safetensors:   5%|▎    | 262M/4.89G [00:05<01:19, 58.1MB/s]\u001b[A^C\n",
-      "model-00001-of-00019.safetensors:   5%|▎    | 262M/4.89G [00:05<01:41, 45.7MB/s]\n",
-      "Downloading shards:   0%|                                | 0/19 [00:05<?, ?it/s]\n",
-      "Traceback (most recent call last):\n",
-      "  File \"/home/user/miniconda/lib/python3.9/runpy.py\", line 197, in _run_module_as_main\n",
-      "    return _run_code(code, main_globals, None,\n",
-      "  File \"/home/user/miniconda/lib/python3.9/runpy.py\", line 87, in _run_code\n",
-      "    exec(code, run_globals)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/__main__.py\", line 461, in <module>\n",
-      "    cli_evaluate()\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/__main__.py\", line 382, in cli_evaluate\n",
-      "    results = evaluator.simple_evaluate(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/utils.py\", line 397, in _wrapper\n",
-      "    return fn(*args, **kwargs)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/evaluator.py\", line 204, in simple_evaluate\n",
-      "    lm = lm_eval.api.registry.get_model(model).create_from_arg_string(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/api/model.py\", line 147, in create_from_arg_string\n",
-      "    return cls(**args, **args2)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/models/huggingface.py\", line 184, in __init__\n",
-      "    self._create_model(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/lm_eval/models/huggingface.py\", line 576, in _create_model\n",
-      "    self._model = self.AUTO_MODEL_CLASS.from_pretrained(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py\", line 564, in from_pretrained\n",
-      "    return model_class.from_pretrained(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/transformers/modeling_utils.py\", line 3990, in from_pretrained\n",
-      "    resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/transformers/utils/hub.py\", line 1098, in get_checkpoint_shard_files\n",
-      "    cached_filename = cached_file(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/transformers/utils/hub.py\", line 403, in cached_file\n",
-      "    resolved_file = hf_hub_download(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py\", line 114, in _inner_fn\n",
-      "    return fn(*args, **kwargs)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/huggingface_hub/file_download.py\", line 862, in hf_hub_download\n",
-      "    return _hf_hub_download_to_cache_dir(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/huggingface_hub/file_download.py\", line 1011, in _hf_hub_download_to_cache_dir\n",
-      "    _download_to_tmp_and_move(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/huggingface_hub/file_download.py\", line 1545, in _download_to_tmp_and_move\n",
-      "    http_get(\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/huggingface_hub/file_download.py\", line 454, in http_get\n",
-      "    for chunk in r.iter_content(chunk_size=constants.DOWNLOAD_CHUNK_SIZE):\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/requests/models.py\", line 820, in generate\n",
-      "    yield from self.raw.stream(chunk_size, decode_content=True)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/urllib3/response.py\", line 576, in stream\n",
-      "    data = self.read(amt=amt, decode_content=decode_content)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/site-packages/urllib3/response.py\", line 519, in read\n",
-      "    data = self._fp.read(amt) if not fp_closed else b\"\"\n",
-      "  File \"/home/user/miniconda/lib/python3.9/http/client.py\", line 455, in read\n",
-      "    n = self.readinto(b)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/http/client.py\", line 499, in readinto\n",
-      "    n = self.fp.readinto(b)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/socket.py\", line 704, in readinto\n",
-      "    return self._sock.recv_into(b)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/ssl.py\", line 1241, in recv_into\n",
-      "    return self.read(nbytes, buffer)\n",
-      "  File \"/home/user/miniconda/lib/python3.9/ssl.py\", line 1099, in read\n",
-      "    return self._sslobj.read(len, buffer)\n",
-      "KeyboardInterrupt\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "!for i in $(echo $TASKS|tr ',' ' '); do accelerate launch -m lm_eval \\\n",
     "--model hf --model_args pretrained=mistralai/Mixtral-8x7B-Instruct-v0.1,revision=41bd4c9e7e4fb318ca40e721131d4933966c2cc1,trust_remote_code=False,dtype=bfloat16,parallelize=True \\\n",