| {"task_name": "arc_challenge", "task_hash": "3059c7ebe96cbc93600127cf3237e8e1", "model_hash": "03418cf8091a9882619950ffb07429a5", "model_config": {"model": "hf_checkpoints/dclm-pool-1b-1x-h-fasttext-open_lm_1b_swiglutorch-warm5000-lr0p003-wd0p033-cd3e-05-bs256-mult1-seed124-tokens28795904000", "revision": null, "trust_remote_code": null, "max_length": 2048, "model_path": null, "model_type": "hf"}, "task_config": {"task_name": "arc_challenge", "task_core": "arc_challenge", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_uncond", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "id", "fewshot_source": "OLMES:ARC-Challenge", "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "use_chat_format": null, "version": 0, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "arc_challenge:rc::olmes"}}, "compute_config": {"batch_size": "16", "max_batch_size": 32, "output_dir": "hf_checkpoints/dclm-pool-1b-1x-h-fasttext-open_lm_1b_swiglutorch-warm5000-lr0p003-wd0p033-cd3e-05-bs256-mult1-seed124-tokens28795904000/olmes_fp32/core_9mcqa", "num_recorded_inputs": 3, "save_raw_requests": true, "recompute_metrics": false, "check_datalake": false, "push_datalake": false, "no_datalake": false}, "processing_time": 59.15019631385803, "current_date": "2024-11-19 21:11:53 UTC", "num_instances": 1172, "beaker_info": {}, "metrics": {"acc_raw": 0.33532423208191126, "acc_per_token": 0.36945392491467577, "acc_per_char": 0.36177474402730375, "correct_loss_raw": 15.167813213612032, "incorrect_loss_raw": 16.08963039536049, "correct_loss_per_token": 2.7816373910187298, "incorrect_loss_per_token": 3.1465591771719446, "correct_loss_per_char": 0.5648754563167282, "incorrect_loss_per_char": 0.6338645292376848, "acc_uncond": 0.3984641638225256, "correct_loss_uncond": -13.69655239866862, "incorrect_loss_uncond": -12.153245893880248, "primary_score": 0.3984641638225256}, "task_idx": 3} |