{"all_primary_scores": ["mmlu:mc::olmes: 0.25455", "mmlu:rc::olmes: 0.333915", "mmlu::olmes: 0.333915", "mmlu_abstract_algebra:mc::olmes: 0.27", "mmlu_anatomy:mc::olmes: 0.2", "mmlu_astronomy:mc::olmes: 0.190789", "mmlu_business_ethics:mc::olmes: 0.2", "mmlu_clinical_knowledge:mc::olmes: 0.211321", "mmlu_college_biology:mc::olmes: 0.201389", "mmlu_college_chemistry:mc::olmes: 0.21", "mmlu_college_computer_science:mc::olmes: 0.34", "mmlu_college_mathematics:mc::olmes: 0.33", "mmlu_college_medicine:mc::olmes: 0.231214", "mmlu_college_physics:mc::olmes: 0.235294", "mmlu_computer_security:mc::olmes: 0.21", "mmlu_conceptual_physics:mc::olmes: 0.331915", "mmlu_econometrics:mc::olmes: 0.210526", "mmlu_electrical_engineering:mc::olmes: 0.22069", "mmlu_elementary_mathematics:mc::olmes: 0.269841", "mmlu_formal_logic:mc::olmes: 0.253968", "mmlu_global_facts:mc::olmes: 0.18", "mmlu_high_school_biology:mc::olmes: 0.290323", "mmlu_high_school_chemistry:mc::olmes: 0.20197", "mmlu_high_school_computer_science:mc::olmes: 0.26", "mmlu_high_school_european_history:mc::olmes: 0.2", "mmlu_high_school_geography:mc::olmes: 0.186869", "mmlu_high_school_government_and_politics:mc::olmes: 0.300518", "mmlu_high_school_macroeconomics:mc::olmes: 0.338462", "mmlu_high_school_mathematics:mc::olmes: 0.255556", "mmlu_high_school_microeconomics:mc::olmes: 0.302521", "mmlu_high_school_physics:mc::olmes: 0.317881", "mmlu_high_school_psychology:mc::olmes: 0.233028", "mmlu_high_school_statistics:mc::olmes: 0.449074", "mmlu_high_school_us_history:mc::olmes: 0.259804", "mmlu_high_school_world_history:mc::olmes: 0.240506", "mmlu_human_aging:mc::olmes: 0.345291", "mmlu_human_sexuality:mc::olmes: 0.259542", "mmlu_international_law:mc::olmes: 0.239669", "mmlu_jurisprudence:mc::olmes: 0.25", "mmlu_logical_fallacies:mc::olmes: 0.251534", "mmlu_machine_learning:mc::olmes: 0.303571", "mmlu_management:mc::olmes: 0.145631", "mmlu_marketing:mc::olmes: 0.299145", "mmlu_medical_genetics:mc::olmes: 0.29", "mmlu_miscellaneous:mc::olmes: 0.273308", "mmlu_moral_disputes:mc::olmes: 0.242775", "mmlu_moral_scenarios:mc::olmes: 0.242458", "mmlu_nutrition:mc::olmes: 0.215686", "mmlu_philosophy:mc::olmes: 0.241158", "mmlu_prehistory:mc::olmes: 0.219136", "mmlu_professional_accounting:mc::olmes: 0.230496", "mmlu_professional_law:mc::olmes: 0.245111", "mmlu_professional_medicine:mc::olmes: 0.441176", "mmlu_professional_psychology:mc::olmes: 0.240196", "mmlu_public_relations:mc::olmes: 0.245455", "mmlu_security_studies:mc::olmes: 0.183673", "mmlu_sociology:mc::olmes: 0.20398", "mmlu_us_foreign_policy:mc::olmes: 0.25", "mmlu_virology:mc::olmes: 0.277108", "mmlu_world_religions:mc::olmes: 0.239766", "mmlu_abstract_algebra:rc::olmes: 0.25", "mmlu_anatomy:rc::olmes: 0.340741", "mmlu_astronomy:rc::olmes: 0.414474", "mmlu_business_ethics:rc::olmes: 0.46", "mmlu_clinical_knowledge:rc::olmes: 0.366038", "mmlu_college_biology:rc::olmes: 0.381944", "mmlu_college_chemistry:rc::olmes: 0.26", "mmlu_college_computer_science:rc::olmes: 0.29", "mmlu_college_mathematics:rc::olmes: 0.19", "mmlu_college_medicine:rc::olmes: 0.289017", "mmlu_college_physics:rc::olmes: 0.245098", "mmlu_computer_security:rc::olmes: 0.43", "mmlu_conceptual_physics:rc::olmes: 0.378723", "mmlu_econometrics:rc::olmes: 0.280702", "mmlu_electrical_engineering:rc::olmes: 0.248276", "mmlu_elementary_mathematics:rc::olmes: 0.275132", "mmlu_formal_logic:rc::olmes: 0.293651", "mmlu_global_facts:rc::olmes: 0.29", "mmlu_high_school_biology:rc::olmes: 0.390323", "mmlu_high_school_chemistry:rc::olmes: 0.216749", "mmlu_high_school_computer_science:rc::olmes: 0.3", "mmlu_high_school_european_history:rc::olmes: 0.448485", "mmlu_high_school_geography:rc::olmes: 0.424242", "mmlu_high_school_government_and_politics:rc::olmes: 0.450777", "mmlu_high_school_macroeconomics:rc::olmes: 0.315385", "mmlu_high_school_mathematics:rc::olmes: 0.162963", "mmlu_high_school_microeconomics:rc::olmes: 0.361345", "mmlu_high_school_physics:rc::olmes: 0.245033", "mmlu_high_school_psychology:rc::olmes: 0.475229", "mmlu_high_school_statistics:rc::olmes: 0.277778", "mmlu_high_school_us_history:rc::olmes: 0.377451", "mmlu_high_school_world_history:rc::olmes: 0.371308", "mmlu_human_aging:rc::olmes: 0.376682", "mmlu_human_sexuality:rc::olmes: 0.396947", "mmlu_international_law:rc::olmes: 0.256198", "mmlu_jurisprudence:rc::olmes: 0.314815", "mmlu_logical_fallacies:rc::olmes: 0.306748", "mmlu_machine_learning:rc::olmes: 0.276786", "mmlu_management:rc::olmes: 0.514563", "mmlu_marketing:rc::olmes: 0.542735", "mmlu_medical_genetics:rc::olmes: 0.43", "mmlu_miscellaneous:rc::olmes: 0.480204", "mmlu_moral_disputes:rc::olmes: 0.251445", "mmlu_moral_scenarios:rc::olmes: 0.240223", "mmlu_nutrition:rc::olmes: 0.326797", "mmlu_philosophy:rc::olmes: 0.302251", "mmlu_prehistory:rc::olmes: 0.342593", "mmlu_professional_accounting:rc::olmes: 0.262411", "mmlu_professional_law:rc::olmes: 0.27575", "mmlu_professional_medicine:rc::olmes: 0.297794", "mmlu_professional_psychology:rc::olmes: 0.320261", "mmlu_public_relations:rc::olmes: 0.290909", "mmlu_security_studies:rc::olmes: 0.273469", "mmlu_sociology:rc::olmes: 0.278607", "mmlu_us_foreign_policy:rc::olmes: 0.37", "mmlu_virology:rc::olmes: 0.301205", "mmlu_world_religions:rc::olmes: 0.502924"], "metrics": [{"task": "mmlu:mc::olmes", "correct_loss_per_token_micro": 1.399370581556837, "correct_loss_per_token_macro": 1.4004924818471334, "incorrect_loss_per_token_micro": 1.4053478479492902, "incorrect_loss_per_token_macro": 1.4081410191080233, "acc_raw_micro": 0.25658738071499787, "acc_raw_macro": 0.25454956238045445, "correct_loss_raw_micro": 1.399370581556837, "correct_loss_raw_macro": 1.4004924818471334, "acc_per_char_micro": 0.25658738071499787, "acc_per_char_macro": 0.25454956238045445, "incorrect_loss_raw_micro": 1.4053478479492902, "incorrect_loss_raw_macro": 1.4081410191080233, "correct_loss_per_char_micro": 0.6996852907784185, "correct_loss_per_char_macro": 0.7002462409235667, "primary_score_micro": 0.25658738071499787, "primary_score_macro": 0.25454956238045445, "incorrect_loss_per_char_micro": 0.7026739239746451, "incorrect_loss_per_char_macro": 0.7040705095540116, "acc_per_token_micro": 0.25658738071499787, "acc_per_token_macro": 0.25454956238045445, "primary_score": 0.25454956238045445, "num_instances": 14042, "task_config": {"task_name": "mmlu:mc::olmes", "task_core": "mmlu_abstract_algebra", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "macro", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"num_tasks": 57, "description": "Aggregate metric", "alias": "mmlu:mc::olmes"}}}, {"task": "mmlu:rc::olmes", "correct_loss_per_token_micro": 2.6432887331562633, "correct_loss_per_token_macro": 2.7731874534749923, "correct_loss_uncond_micro": -15.04736503639418, "correct_loss_uncond_macro": -13.507832404085681, "incorrect_loss_uncond_micro": -14.061741272932458, "incorrect_loss_uncond_macro": -12.577336096784808, "incorrect_loss_per_token_micro": 2.956806828783771, "incorrect_loss_per_token_macro": 3.083547324785819, "acc_raw_micro": 0.317048853439681, "acc_raw_macro": 0.3165782466450824, "correct_loss_raw_micro": 21.79074947781205, "correct_loss_raw_macro": 20.761361040112263, "acc_per_char_micro": 0.33150548354935194, "acc_per_char_macro": 0.333915463567101, "incorrect_loss_raw_micro": 22.025241633518036, "incorrect_loss_raw_macro": 20.90722468526756, "correct_loss_per_char_micro": 0.622242090275648, "correct_loss_per_char_macro": 0.6912111226964194, "primary_score_micro": 0.33150548354935194, "primary_score_macro": 0.333915463567101, "incorrect_loss_per_char_micro": 0.6791941747100239, "incorrect_loss_per_char_macro": 0.7448731712983696, "acc_uncond_micro": 0.3470303375587523, "acc_uncond_macro": 0.34845882687855845, "acc_per_token_micro": 0.3336419313488107, "acc_per_token_macro": 0.3358986998185535, "primary_score": 0.333915463567101, "num_instances": 14042, "task_config": {"task_name": "mmlu:rc::olmes", "task_core": "mmlu_abstract_algebra", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "macro", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"num_tasks": 57, "description": "Aggregate metric", "alias": "mmlu:rc::olmes"}}}, {"task": "mmlu::olmes", "primary_score": 0.333915463567101, "num_instances": 28084, "task_config": {"task_name": "mmlu::olmes", "task_core": "mmlu_abstract_algebra", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "mc_or_rc", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"num_tasks": 2, "description": "Best of MC vs RC", "used_mc_or_rc": "rc", "alias": "mmlu::olmes"}}}, {"task": "mmlu_abstract_algebra:mc", "acc_raw": 0.27, "acc_per_token": 0.27, "acc_per_char": 0.27, "correct_loss_raw": 1.5122956645488739, "incorrect_loss_raw": 1.508113094369571, "correct_loss_per_token": 1.5122956645488739, "incorrect_loss_per_token": 1.508113094369571, "correct_loss_per_char": 0.7561478322744369, "incorrect_loss_per_char": 0.7540565471847855, "primary_score": 0.27, "num_instances": 100, "task_config": {"task_name": "mmlu_abstract_algebra:mc", "task_core": "mmlu_abstract_algebra", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_abstract_algebra:mc::olmes"}}}, {"task": "mmlu_anatomy:mc", "acc_raw": 0.2, "acc_per_token": 0.2, "acc_per_char": 0.2, "correct_loss_raw": 1.405913030659711, "incorrect_loss_raw": 1.394696780192999, "correct_loss_per_token": 1.405913030659711, "incorrect_loss_per_token": 1.394696780192999, "correct_loss_per_char": 0.7029565153298555, "incorrect_loss_per_char": 0.6973483900964995, "primary_score": 0.2, "num_instances": 135, "task_config": {"task_name": "mmlu_anatomy:mc", "task_core": "mmlu_anatomy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "anatomy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_anatomy:mc::olmes"}}}, {"task": "mmlu_astronomy:mc", "acc_raw": 0.19078947368421054, "acc_per_token": 0.19078947368421054, "acc_per_char": 0.19078947368421054, "correct_loss_raw": 1.4413342103362083, "incorrect_loss_raw": 1.3992736855881265, "correct_loss_per_token": 1.4413342103362083, "incorrect_loss_per_token": 1.3992736855881265, "correct_loss_per_char": 0.7206671051681042, "incorrect_loss_per_char": 0.6996368427940632, "primary_score": 0.19078947368421054, "num_instances": 152, "task_config": {"task_name": "mmlu_astronomy:mc", "task_core": "mmlu_astronomy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "astronomy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_astronomy:mc::olmes"}}}, {"task": "mmlu_business_ethics:mc", "acc_raw": 0.2, "acc_per_token": 0.2, "acc_per_char": 0.2, "correct_loss_raw": 1.4059372037649154, "incorrect_loss_raw": 1.4068191585938135, "correct_loss_per_token": 1.4059372037649154, "incorrect_loss_per_token": 1.4068191585938135, "correct_loss_per_char": 0.7029686018824577, "incorrect_loss_per_char": 0.7034095792969067, "primary_score": 0.2, "num_instances": 100, "task_config": {"task_name": "mmlu_business_ethics:mc", "task_core": "mmlu_business_ethics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "business_ethics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_business_ethics:mc::olmes"}}}, {"task": "mmlu_clinical_knowledge:mc", "acc_raw": 0.21132075471698114, "acc_per_token": 0.21132075471698114, "acc_per_char": 0.21132075471698114, "correct_loss_raw": 1.4041231951623592, "incorrect_loss_raw": 1.3956608722044994, "correct_loss_per_token": 1.4041231951623592, "incorrect_loss_per_token": 1.3956608722044994, "correct_loss_per_char": 0.7020615975811796, "incorrect_loss_per_char": 0.6978304361022497, "primary_score": 0.21132075471698114, "num_instances": 265, "task_config": {"task_name": "mmlu_clinical_knowledge:mc", "task_core": "mmlu_clinical_knowledge", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "clinical_knowledge", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_clinical_knowledge:mc::olmes"}}}, {"task": "mmlu_college_biology:mc", "acc_raw": 0.2013888888888889, "acc_per_token": 0.2013888888888889, "acc_per_char": 0.2013888888888889, "correct_loss_raw": 1.3990219914250903, "incorrect_loss_raw": 1.4011454527024867, "correct_loss_per_token": 1.3990219914250903, "incorrect_loss_per_token": 1.4011454527024867, "correct_loss_per_char": 0.6995109957125452, "incorrect_loss_per_char": 0.7005727263512433, "primary_score": 0.2013888888888889, "num_instances": 144, "task_config": {"task_name": "mmlu_college_biology:mc", "task_core": "mmlu_college_biology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_biology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_biology:mc::olmes"}}}, {"task": "mmlu_college_chemistry:mc", "acc_raw": 0.21, "acc_per_token": 0.21, "acc_per_char": 0.21, "correct_loss_raw": 1.4002672040462494, "incorrect_loss_raw": 1.3961779685815174, "correct_loss_per_token": 1.4002672040462494, "incorrect_loss_per_token": 1.3961779685815174, "correct_loss_per_char": 0.7001336020231247, "incorrect_loss_per_char": 0.6980889842907587, "primary_score": 0.21, "num_instances": 100, "task_config": {"task_name": "mmlu_college_chemistry:mc", "task_core": "mmlu_college_chemistry", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_chemistry", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_chemistry:mc::olmes"}}}, {"task": "mmlu_college_computer_science:mc", "acc_raw": 0.34, "acc_per_token": 0.34, "acc_per_char": 0.34, "correct_loss_raw": 1.3831730151176453, "incorrect_loss_raw": 1.4183711288372678, "correct_loss_per_token": 1.3831730151176453, "incorrect_loss_per_token": 1.4183711288372678, "correct_loss_per_char": 0.6915865075588227, "incorrect_loss_per_char": 0.7091855644186339, "primary_score": 0.34, "num_instances": 100, "task_config": {"task_name": "mmlu_college_computer_science:mc", "task_core": "mmlu_college_computer_science", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_computer_science", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_computer_science:mc::olmes"}}}, {"task": "mmlu_college_mathematics:mc", "acc_raw": 0.33, "acc_per_token": 0.33, "acc_per_char": 0.33, "correct_loss_raw": 1.3795828753709793, "incorrect_loss_raw": 1.4324306764205301, "correct_loss_per_token": 1.3795828753709793, "incorrect_loss_per_token": 1.4324306764205301, "correct_loss_per_char": 0.6897914376854897, "incorrect_loss_per_char": 0.7162153382102651, "primary_score": 0.33, "num_instances": 100, "task_config": {"task_name": "mmlu_college_mathematics:mc", "task_core": "mmlu_college_mathematics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_mathematics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_mathematics:mc::olmes"}}}, {"task": "mmlu_college_medicine:mc", "acc_raw": 0.23121387283236994, "acc_per_token": 0.23121387283236994, "acc_per_char": 0.23121387283236994, "correct_loss_raw": 1.4082309800076347, "incorrect_loss_raw": 1.3981844719664442, "correct_loss_per_token": 1.4082309800076347, "incorrect_loss_per_token": 1.3981844719664442, "correct_loss_per_char": 0.7041154900038173, "incorrect_loss_per_char": 0.6990922359832221, "primary_score": 0.23121387283236994, "num_instances": 173, "task_config": {"task_name": "mmlu_college_medicine:mc", "task_core": "mmlu_college_medicine", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_medicine", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_medicine:mc::olmes"}}}, {"task": "mmlu_college_physics:mc", "acc_raw": 0.23529411764705882, "acc_per_token": 0.23529411764705882, "acc_per_char": 0.23529411764705882, "correct_loss_raw": 1.3708963253918816, "incorrect_loss_raw": 1.408653272522821, "correct_loss_per_token": 1.3708963253918816, "incorrect_loss_per_token": 1.408653272522821, "correct_loss_per_char": 0.6854481626959408, "incorrect_loss_per_char": 0.7043266362614105, "primary_score": 0.23529411764705882, "num_instances": 102, "task_config": {"task_name": "mmlu_college_physics:mc", "task_core": "mmlu_college_physics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_physics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_physics:mc::olmes"}}}, {"task": "mmlu_computer_security:mc", "acc_raw": 0.21, "acc_per_token": 0.21, "acc_per_char": 0.21, "correct_loss_raw": 1.4048889875411987, "incorrect_loss_raw": 1.404950787623724, "correct_loss_per_token": 1.4048889875411987, "incorrect_loss_per_token": 1.404950787623724, "correct_loss_per_char": 0.7024444937705994, "incorrect_loss_per_char": 0.702475393811862, "primary_score": 0.21, "num_instances": 100, "task_config": {"task_name": "mmlu_computer_security:mc", "task_core": "mmlu_computer_security", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "computer_security", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_computer_security:mc::olmes"}}}, {"task": "mmlu_conceptual_physics:mc", "acc_raw": 0.33191489361702126, "acc_per_token": 0.33191489361702126, "acc_per_char": 0.33191489361702126, "correct_loss_raw": 1.3766544839169117, "incorrect_loss_raw": 1.4139737217138846, "correct_loss_per_token": 1.3766544839169117, "incorrect_loss_per_token": 1.4139737217138846, "correct_loss_per_char": 0.6883272419584558, "incorrect_loss_per_char": 0.7069868608569423, "primary_score": 0.33191489361702126, "num_instances": 235, "task_config": {"task_name": "mmlu_conceptual_physics:mc", "task_core": "mmlu_conceptual_physics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "conceptual_physics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_conceptual_physics:mc::olmes"}}}, {"task": "mmlu_econometrics:mc", "acc_raw": 0.21052631578947367, "acc_per_token": 0.21052631578947367, "acc_per_char": 0.21052631578947367, "correct_loss_raw": 1.4257322455707349, "incorrect_loss_raw": 1.4038150355829833, "correct_loss_per_token": 1.4257322455707349, "incorrect_loss_per_token": 1.4038150355829833, "correct_loss_per_char": 0.7128661227853674, "incorrect_loss_per_char": 0.7019075177914916, "primary_score": 0.21052631578947367, "num_instances": 114, "task_config": {"task_name": "mmlu_econometrics:mc", "task_core": "mmlu_econometrics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "econometrics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_econometrics:mc::olmes"}}}, {"task": "mmlu_electrical_engineering:mc", "acc_raw": 0.2206896551724138, "acc_per_token": 0.2206896551724138, "acc_per_char": 0.2206896551724138, "correct_loss_raw": 1.4267057862775079, "incorrect_loss_raw": 1.3974592317109829, "correct_loss_per_token": 1.4267057862775079, "incorrect_loss_per_token": 1.3974592317109829, "correct_loss_per_char": 0.7133528931387539, "incorrect_loss_per_char": 0.6987296158554914, "primary_score": 0.2206896551724138, "num_instances": 145, "task_config": {"task_name": "mmlu_electrical_engineering:mc", "task_core": "mmlu_electrical_engineering", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "electrical_engineering", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_electrical_engineering:mc::olmes"}}}, {"task": "mmlu_elementary_mathematics:mc", "acc_raw": 0.2698412698412698, "acc_per_token": 0.2698412698412698, "acc_per_char": 0.2698412698412698, "correct_loss_raw": 1.40157401782495, "incorrect_loss_raw": 1.3962224588928183, "correct_loss_per_token": 1.40157401782495, "incorrect_loss_per_token": 1.3962224588928183, "correct_loss_per_char": 0.700787008912475, "incorrect_loss_per_char": 0.6981112294464091, "primary_score": 0.2698412698412698, "num_instances": 378, "task_config": {"task_name": "mmlu_elementary_mathematics:mc", "task_core": "mmlu_elementary_mathematics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "elementary_mathematics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_elementary_mathematics:mc::olmes"}}}, {"task": "mmlu_formal_logic:mc", "acc_raw": 0.25396825396825395, "acc_per_token": 0.25396825396825395, "acc_per_char": 0.25396825396825395, "correct_loss_raw": 1.4213087558746338, "incorrect_loss_raw": 1.4132544647448908, "correct_loss_per_token": 1.4213087558746338, "incorrect_loss_per_token": 1.4132544647448908, "correct_loss_per_char": 0.7106543779373169, "incorrect_loss_per_char": 0.7066272323724454, "primary_score": 0.25396825396825395, "num_instances": 126, "task_config": {"task_name": "mmlu_formal_logic:mc", "task_core": "mmlu_formal_logic", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "formal_logic", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_formal_logic:mc::olmes"}}}, {"task": "mmlu_global_facts:mc", "acc_raw": 0.18, "acc_per_token": 0.18, "acc_per_char": 0.18, "correct_loss_raw": 1.4756081253290176, "incorrect_loss_raw": 1.4302349406480783, "correct_loss_per_token": 1.4756081253290176, "incorrect_loss_per_token": 1.4302349406480783, "correct_loss_per_char": 0.7378040626645088, "incorrect_loss_per_char": 0.7151174703240392, "primary_score": 0.18, "num_instances": 100, "task_config": {"task_name": "mmlu_global_facts:mc", "task_core": "mmlu_global_facts", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "global_facts", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_global_facts:mc::olmes"}}}, {"task": "mmlu_high_school_biology:mc", "acc_raw": 0.2903225806451613, "acc_per_token": 0.2903225806451613, "acc_per_char": 0.2903225806451613, "correct_loss_raw": 1.3885792163110549, "incorrect_loss_raw": 1.40228969813675, "correct_loss_per_token": 1.3885792163110549, "incorrect_loss_per_token": 1.40228969813675, "correct_loss_per_char": 0.6942896081555274, "incorrect_loss_per_char": 0.701144849068375, "primary_score": 0.2903225806451613, "num_instances": 310, "task_config": {"task_name": "mmlu_high_school_biology:mc", "task_core": "mmlu_high_school_biology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_biology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_biology:mc::olmes"}}}, {"task": "mmlu_high_school_chemistry:mc", "acc_raw": 0.2019704433497537, "acc_per_token": 0.2019704433497537, "acc_per_char": 0.2019704433497537, "correct_loss_raw": 1.3932797168863231, "incorrect_loss_raw": 1.3984861059729097, "correct_loss_per_token": 1.3932797168863231, "incorrect_loss_per_token": 1.3984861059729097, "correct_loss_per_char": 0.6966398584431616, "incorrect_loss_per_char": 0.6992430529864548, "primary_score": 0.2019704433497537, "num_instances": 203, "task_config": {"task_name": "mmlu_high_school_chemistry:mc", "task_core": "mmlu_high_school_chemistry", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_chemistry", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_chemistry:mc::olmes"}}}, {"task": "mmlu_high_school_computer_science:mc", "acc_raw": 0.26, "acc_per_token": 0.26, "acc_per_char": 0.26, "correct_loss_raw": 1.3843735754489899, "incorrect_loss_raw": 1.4085347572962434, "correct_loss_per_token": 1.3843735754489899, "incorrect_loss_per_token": 1.4085347572962434, "correct_loss_per_char": 0.6921867877244949, "incorrect_loss_per_char": 0.7042673786481217, "primary_score": 0.26, "num_instances": 100, "task_config": {"task_name": "mmlu_high_school_computer_science:mc", "task_core": "mmlu_high_school_computer_science", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_computer_science", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_computer_science:mc::olmes"}}}, {"task": "mmlu_high_school_european_history:mc", "acc_raw": 0.2, "acc_per_token": 0.2, "acc_per_char": 0.2, "correct_loss_raw": 1.4093764514634104, "incorrect_loss_raw": 1.3966222804002089, "correct_loss_per_token": 1.4093764514634104, "incorrect_loss_per_token": 1.3966222804002089, "correct_loss_per_char": 0.7046882257317052, "incorrect_loss_per_char": 0.6983111402001044, "primary_score": 0.2, "num_instances": 165, "task_config": {"task_name": "mmlu_high_school_european_history:mc", "task_core": "mmlu_high_school_european_history", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_european_history", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_european_history:mc::olmes"}}}, {"task": "mmlu_high_school_geography:mc", "acc_raw": 0.18686868686868688, "acc_per_token": 0.18686868686868688, "acc_per_char": 0.18686868686868688, "correct_loss_raw": 1.4153487002006684, "incorrect_loss_raw": 1.3935150254254387, "correct_loss_per_token": 1.4153487002006684, "incorrect_loss_per_token": 1.3935150254254387, "correct_loss_per_char": 0.7076743501003342, "incorrect_loss_per_char": 0.6967575127127194, "primary_score": 0.18686868686868688, "num_instances": 198, "task_config": {"task_name": "mmlu_high_school_geography:mc", "task_core": "mmlu_high_school_geography", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_geography", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_geography:mc::olmes"}}}, {"task": "mmlu_high_school_government_and_politics:mc", "acc_raw": 0.3005181347150259, "acc_per_token": 0.3005181347150259, "acc_per_char": 0.3005181347150259, "correct_loss_raw": 1.3870787317888724, "incorrect_loss_raw": 1.4027198924713595, "correct_loss_per_token": 1.3870787317888724, "incorrect_loss_per_token": 1.4027198924713595, "correct_loss_per_char": 0.6935393658944362, "incorrect_loss_per_char": 0.7013599462356798, "primary_score": 0.3005181347150259, "num_instances": 193, "task_config": {"task_name": "mmlu_high_school_government_and_politics:mc", "task_core": "mmlu_high_school_government_and_politics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_government_and_politics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_government_and_politics:mc::olmes"}}}, {"task": "mmlu_high_school_macroeconomics:mc", "acc_raw": 0.3384615384615385, "acc_per_token": 0.3384615384615385, "acc_per_char": 0.3384615384615385, "correct_loss_raw": 1.3830373278030983, "incorrect_loss_raw": 1.4065138395525443, "correct_loss_per_token": 1.3830373278030983, "incorrect_loss_per_token": 1.4065138395525443, "correct_loss_per_char": 0.6915186639015491, "incorrect_loss_per_char": 0.7032569197762721, "primary_score": 0.3384615384615385, "num_instances": 390, "task_config": {"task_name": "mmlu_high_school_macroeconomics:mc", "task_core": "mmlu_high_school_macroeconomics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_macroeconomics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_macroeconomics:mc::olmes"}}}, {"task": "mmlu_high_school_mathematics:mc", "acc_raw": 0.25555555555555554, "acc_per_token": 0.25555555555555554, "acc_per_char": 0.25555555555555554, "correct_loss_raw": 1.425081949322312, "incorrect_loss_raw": 1.418366907702551, "correct_loss_per_token": 1.425081949322312, "incorrect_loss_per_token": 1.418366907702551, "correct_loss_per_char": 0.712540974661156, "incorrect_loss_per_char": 0.7091834538512755, "primary_score": 0.25555555555555554, "num_instances": 270, "task_config": {"task_name": "mmlu_high_school_mathematics:mc", "task_core": "mmlu_high_school_mathematics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_mathematics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_mathematics:mc::olmes"}}}, {"task": "mmlu_high_school_microeconomics:mc", "acc_raw": 0.3025210084033613, "acc_per_token": 0.3025210084033613, "acc_per_char": 0.3025210084033613, "correct_loss_raw": 1.3908371038797522, "incorrect_loss_raw": 1.399872533580502, "correct_loss_per_token": 1.3908371038797522, "incorrect_loss_per_token": 1.399872533580502, "correct_loss_per_char": 0.6954185519398761, "incorrect_loss_per_char": 0.699936266790251, "primary_score": 0.3025210084033613, "num_instances": 238, "task_config": {"task_name": "mmlu_high_school_microeconomics:mc", "task_core": "mmlu_high_school_microeconomics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_microeconomics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_microeconomics:mc::olmes"}}}, {"task": "mmlu_high_school_physics:mc", "acc_raw": 0.31788079470198677, "acc_per_token": 0.31788079470198677, "acc_per_char": 0.31788079470198677, "correct_loss_raw": 1.3750574825615283, "incorrect_loss_raw": 1.4153292002793698, "correct_loss_per_token": 1.3750574825615283, "incorrect_loss_per_token": 1.4153292002793698, "correct_loss_per_char": 0.6875287412807641, "incorrect_loss_per_char": 0.7076646001396849, "primary_score": 0.31788079470198677, "num_instances": 151, "task_config": {"task_name": "mmlu_high_school_physics:mc", "task_core": "mmlu_high_school_physics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_physics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_physics:mc::olmes"}}}, {"task": "mmlu_high_school_psychology:mc", "acc_raw": 0.23302752293577983, "acc_per_token": 0.23302752293577983, "acc_per_char": 0.23302752293577983, "correct_loss_raw": 1.4060063779900926, "incorrect_loss_raw": 1.3952868980005249, "correct_loss_per_token": 1.4060063779900926, "incorrect_loss_per_token": 1.3952868980005249, "correct_loss_per_char": 0.7030031889950463, "incorrect_loss_per_char": 0.6976434490002624, "primary_score": 0.23302752293577983, "num_instances": 545, "task_config": {"task_name": "mmlu_high_school_psychology:mc", "task_core": "mmlu_high_school_psychology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_psychology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_psychology:mc::olmes"}}}, {"task": "mmlu_high_school_statistics:mc", "acc_raw": 0.44907407407407407, "acc_per_token": 0.44907407407407407, "acc_per_char": 0.44907407407407407, "correct_loss_raw": 1.32589447553511, "incorrect_loss_raw": 1.451398033786703, "correct_loss_per_token": 1.32589447553511, "incorrect_loss_per_token": 1.451398033786703, "correct_loss_per_char": 0.662947237767555, "incorrect_loss_per_char": 0.7256990168933515, "primary_score": 0.44907407407407407, "num_instances": 216, "task_config": {"task_name": "mmlu_high_school_statistics:mc", "task_core": "mmlu_high_school_statistics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_statistics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_statistics:mc::olmes"}}}, {"task": "mmlu_high_school_us_history:mc", "acc_raw": 0.25980392156862747, "acc_per_token": 0.25980392156862747, "acc_per_char": 0.25980392156862747, "correct_loss_raw": 1.3946236991414838, "incorrect_loss_raw": 1.4083286948453368, "correct_loss_per_token": 1.3946236991414838, "incorrect_loss_per_token": 1.4083286948453368, "correct_loss_per_char": 0.6973118495707419, "incorrect_loss_per_char": 0.7041643474226684, "primary_score": 0.25980392156862747, "num_instances": 204, "task_config": {"task_name": "mmlu_high_school_us_history:mc", "task_core": "mmlu_high_school_us_history", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_us_history", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_us_history:mc::olmes"}}}, {"task": "mmlu_high_school_world_history:mc", "acc_raw": 0.24050632911392406, "acc_per_token": 0.24050632911392406, "acc_per_char": 0.24050632911392406, "correct_loss_raw": 1.389097487876184, "incorrect_loss_raw": 1.3955407057130385, "correct_loss_per_token": 1.389097487876184, "incorrect_loss_per_token": 1.3955407057130385, "correct_loss_per_char": 0.694548743938092, "incorrect_loss_per_char": 0.6977703528565192, "primary_score": 0.24050632911392406, "num_instances": 237, "task_config": {"task_name": "mmlu_high_school_world_history:mc", "task_core": "mmlu_high_school_world_history", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_world_history", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_world_history:mc::olmes"}}}, {"task": "mmlu_human_aging:mc", "acc_raw": 0.3452914798206278, "acc_per_token": 0.3452914798206278, "acc_per_char": 0.3452914798206278, "correct_loss_raw": 1.3652815257487276, "incorrect_loss_raw": 1.4116136404074542, "correct_loss_per_token": 1.3652815257487276, "incorrect_loss_per_token": 1.4116136404074542, "correct_loss_per_char": 0.6826407628743638, "incorrect_loss_per_char": 0.7058068202037271, "primary_score": 0.3452914798206278, "num_instances": 223, "task_config": {"task_name": "mmlu_human_aging:mc", "task_core": "mmlu_human_aging", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "human_aging", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_human_aging:mc::olmes"}}}, {"task": "mmlu_human_sexuality:mc", "acc_raw": 0.2595419847328244, "acc_per_token": 0.2595419847328244, "acc_per_char": 0.2595419847328244, "correct_loss_raw": 1.4200853368708195, "incorrect_loss_raw": 1.4141683085577486, "correct_loss_per_token": 1.4200853368708195, "incorrect_loss_per_token": 1.4141683085577486, "correct_loss_per_char": 0.7100426684354098, "incorrect_loss_per_char": 0.7070841542788743, "primary_score": 0.2595419847328244, "num_instances": 131, "task_config": {"task_name": "mmlu_human_sexuality:mc", "task_core": "mmlu_human_sexuality", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "human_sexuality", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_human_sexuality:mc::olmes"}}}, {"task": "mmlu_international_law:mc", "acc_raw": 0.2396694214876033, "acc_per_token": 0.2396694214876033, "acc_per_char": 0.2396694214876033, "correct_loss_raw": 1.3773366005952694, "incorrect_loss_raw": 1.4053297351542908, "correct_loss_per_token": 1.3773366005952694, "incorrect_loss_per_token": 1.4053297351542908, "correct_loss_per_char": 0.6886683002976347, "incorrect_loss_per_char": 0.7026648675771454, "primary_score": 0.2396694214876033, "num_instances": 121, "task_config": {"task_name": "mmlu_international_law:mc", "task_core": "mmlu_international_law", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "international_law", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_international_law:mc::olmes"}}}, {"task": "mmlu_jurisprudence:mc", "acc_raw": 0.25, "acc_per_token": 0.25, "acc_per_char": 0.25, "correct_loss_raw": 1.4049190200037427, "incorrect_loss_raw": 1.400600313220495, "correct_loss_per_token": 1.4049190200037427, "incorrect_loss_per_token": 1.400600313220495, "correct_loss_per_char": 0.7024595100018713, "incorrect_loss_per_char": 0.7003001566102475, "primary_score": 0.25, "num_instances": 108, "task_config": {"task_name": "mmlu_jurisprudence:mc", "task_core": "mmlu_jurisprudence", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "jurisprudence", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_jurisprudence:mc::olmes"}}}, {"task": "mmlu_logical_fallacies:mc", "acc_raw": 0.25153374233128833, "acc_per_token": 0.25153374233128833, "acc_per_char": 0.25153374233128833, "correct_loss_raw": 1.3859041256407287, "incorrect_loss_raw": 1.4001630442274127, "correct_loss_per_token": 1.3859041256407287, "incorrect_loss_per_token": 1.4001630442274127, "correct_loss_per_char": 0.6929520628203644, "incorrect_loss_per_char": 0.7000815221137063, "primary_score": 0.25153374233128833, "num_instances": 163, "task_config": {"task_name": "mmlu_logical_fallacies:mc", "task_core": "mmlu_logical_fallacies", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "logical_fallacies", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_logical_fallacies:mc::olmes"}}}, {"task": "mmlu_machine_learning:mc", "acc_raw": 0.30357142857142855, "acc_per_token": 0.30357142857142855, "acc_per_char": 0.30357142857142855, "correct_loss_raw": 1.3839700307164873, "incorrect_loss_raw": 1.4140378013253208, "correct_loss_per_token": 1.3839700307164873, "incorrect_loss_per_token": 1.4140378013253208, "correct_loss_per_char": 0.6919850153582436, "incorrect_loss_per_char": 0.7070189006626604, "primary_score": 0.30357142857142855, "num_instances": 112, "task_config": {"task_name": "mmlu_machine_learning:mc", "task_core": "mmlu_machine_learning", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "machine_learning", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_machine_learning:mc::olmes"}}}, {"task": "mmlu_management:mc", "acc_raw": 0.14563106796116504, "acc_per_token": 0.14563106796116504, "acc_per_char": 0.14563106796116504, "correct_loss_raw": 1.4238631702163844, "incorrect_loss_raw": 1.3957783520028815, "correct_loss_per_token": 1.4238631702163844, "incorrect_loss_per_token": 1.3957783520028815, "correct_loss_per_char": 0.7119315851081922, "incorrect_loss_per_char": 0.6978891760014407, "primary_score": 0.14563106796116504, "num_instances": 103, "task_config": {"task_name": "mmlu_management:mc", "task_core": "mmlu_management", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "management", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_management:mc::olmes"}}}, {"task": "mmlu_marketing:mc", "acc_raw": 0.29914529914529914, "acc_per_token": 0.29914529914529914, "acc_per_char": 0.29914529914529914, "correct_loss_raw": 1.3953857554329767, "incorrect_loss_raw": 1.410645197174828, "correct_loss_per_token": 1.3953857554329767, "incorrect_loss_per_token": 1.410645197174828, "correct_loss_per_char": 0.6976928777164884, "incorrect_loss_per_char": 0.705322598587414, "primary_score": 0.29914529914529914, "num_instances": 234, "task_config": {"task_name": "mmlu_marketing:mc", "task_core": "mmlu_marketing", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "marketing", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_marketing:mc::olmes"}}}, {"task": "mmlu_medical_genetics:mc", "acc_raw": 0.29, "acc_per_token": 0.29, "acc_per_char": 0.29, "correct_loss_raw": 1.3824806594848633, "incorrect_loss_raw": 1.4173768124977746, "correct_loss_per_token": 1.3824806594848633, "incorrect_loss_per_token": 1.4173768124977746, "correct_loss_per_char": 0.6912403297424317, "incorrect_loss_per_char": 0.7086884062488873, "primary_score": 0.29, "num_instances": 100, "task_config": {"task_name": "mmlu_medical_genetics:mc", "task_core": "mmlu_medical_genetics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "medical_genetics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_medical_genetics:mc::olmes"}}}, {"task": "mmlu_miscellaneous:mc", "acc_raw": 0.27330779054916987, "acc_per_token": 0.27330779054916987, "acc_per_char": 0.27330779054916987, "correct_loss_raw": 1.3916616130758215, "incorrect_loss_raw": 1.4047548287013272, "correct_loss_per_token": 1.3916616130758215, "incorrect_loss_per_token": 1.4047548287013272, "correct_loss_per_char": 0.6958308065379107, "incorrect_loss_per_char": 0.7023774143506636, "primary_score": 0.27330779054916987, "num_instances": 783, "task_config": {"task_name": "mmlu_miscellaneous:mc", "task_core": "mmlu_miscellaneous", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "miscellaneous", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_miscellaneous:mc::olmes"}}}, {"task": "mmlu_moral_disputes:mc", "acc_raw": 0.24277456647398843, "acc_per_token": 0.24277456647398843, "acc_per_char": 0.24277456647398843, "correct_loss_raw": 1.4040873453796254, "incorrect_loss_raw": 1.4014373959144413, "correct_loss_per_token": 1.4040873453796254, "incorrect_loss_per_token": 1.4014373959144413, "correct_loss_per_char": 0.7020436726898127, "incorrect_loss_per_char": 0.7007186979572206, "primary_score": 0.24277456647398843, "num_instances": 346, "task_config": {"task_name": "mmlu_moral_disputes:mc", "task_core": "mmlu_moral_disputes", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "moral_disputes", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_moral_disputes:mc::olmes"}}}, {"task": "mmlu_moral_scenarios:mc", "acc_raw": 0.2424581005586592, "acc_per_token": 0.2424581005586592, "acc_per_char": 0.2424581005586592, "correct_loss_raw": 1.4125996712199802, "incorrect_loss_raw": 1.4057185177474494, "correct_loss_per_token": 1.4125996712199802, "incorrect_loss_per_token": 1.4057185177474494, "correct_loss_per_char": 0.7062998356099901, "incorrect_loss_per_char": 0.7028592588737247, "primary_score": 0.2424581005586592, "num_instances": 895, "task_config": {"task_name": "mmlu_moral_scenarios:mc", "task_core": "mmlu_moral_scenarios", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "moral_scenarios", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_moral_scenarios:mc::olmes"}}}, {"task": "mmlu_nutrition:mc", "acc_raw": 0.21568627450980393, "acc_per_token": 0.21568627450980393, "acc_per_char": 0.21568627450980393, "correct_loss_raw": 1.4100196493996515, "incorrect_loss_raw": 1.3950737732947516, "correct_loss_per_token": 1.4100196493996515, "incorrect_loss_per_token": 1.3950737732947516, "correct_loss_per_char": 0.7050098246998258, "incorrect_loss_per_char": 0.6975368866473758, "primary_score": 0.21568627450980393, "num_instances": 306, "task_config": {"task_name": "mmlu_nutrition:mc", "task_core": "mmlu_nutrition", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "nutrition", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_nutrition:mc::olmes"}}}, {"task": "mmlu_philosophy:mc", "acc_raw": 0.24115755627009647, "acc_per_token": 0.24115755627009647, "acc_per_char": 0.24115755627009647, "correct_loss_raw": 1.3945161211528962, "incorrect_loss_raw": 1.3982760004164196, "correct_loss_per_token": 1.3945161211528962, "incorrect_loss_per_token": 1.3982760004164196, "correct_loss_per_char": 0.6972580605764481, "incorrect_loss_per_char": 0.6991380002082098, "primary_score": 0.24115755627009647, "num_instances": 311, "task_config": {"task_name": "mmlu_philosophy:mc", "task_core": "mmlu_philosophy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "philosophy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_philosophy:mc::olmes"}}}, {"task": "mmlu_prehistory:mc", "acc_raw": 0.2191358024691358, "acc_per_token": 0.2191358024691358, "acc_per_char": 0.2191358024691358, "correct_loss_raw": 1.409603499703937, "incorrect_loss_raw": 1.4012677543815766, "correct_loss_per_token": 1.409603499703937, "incorrect_loss_per_token": 1.4012677543815766, "correct_loss_per_char": 0.7048017498519685, "incorrect_loss_per_char": 0.7006338771907883, "primary_score": 0.2191358024691358, "num_instances": 324, "task_config": {"task_name": "mmlu_prehistory:mc", "task_core": "mmlu_prehistory", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "prehistory", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_prehistory:mc::olmes"}}}, {"task": "mmlu_professional_accounting:mc", "acc_raw": 0.23049645390070922, "acc_per_token": 0.23049645390070922, "acc_per_char": 0.23049645390070922, "correct_loss_raw": 1.4010881396895605, "incorrect_loss_raw": 1.3946880787681466, "correct_loss_per_token": 1.4010881396895605, "incorrect_loss_per_token": 1.3946880787681466, "correct_loss_per_char": 0.7005440698447802, "incorrect_loss_per_char": 0.6973440393840733, "primary_score": 0.23049645390070922, "num_instances": 282, "task_config": {"task_name": "mmlu_professional_accounting:mc", "task_core": "mmlu_professional_accounting", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_accounting", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_accounting:mc::olmes"}}}, {"task": "mmlu_professional_law:mc", "acc_raw": 0.24511082138200782, "acc_per_token": 0.24511082138200782, "acc_per_char": 0.24511082138200782, "correct_loss_raw": 1.4002322044801525, "incorrect_loss_raw": 1.3988230905031325, "correct_loss_per_token": 1.4002322044801525, "incorrect_loss_per_token": 1.3988230905031325, "correct_loss_per_char": 0.7001161022400763, "incorrect_loss_per_char": 0.6994115452515662, "primary_score": 0.24511082138200782, "num_instances": 1534, "task_config": {"task_name": "mmlu_professional_law:mc", "task_core": "mmlu_professional_law", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_law", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_law:mc::olmes"}}}, {"task": "mmlu_professional_medicine:mc", "acc_raw": 0.4411764705882353, "acc_per_token": 0.4411764705882353, "acc_per_char": 0.4411764705882353, "correct_loss_raw": 1.3388401828706264, "incorrect_loss_raw": 1.446392762748634, "correct_loss_per_token": 1.3388401828706264, "incorrect_loss_per_token": 1.446392762748634, "correct_loss_per_char": 0.6694200914353132, "incorrect_loss_per_char": 0.723196381374317, "primary_score": 0.4411764705882353, "num_instances": 272, "task_config": {"task_name": "mmlu_professional_medicine:mc", "task_core": "mmlu_professional_medicine", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_medicine", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_medicine:mc::olmes"}}}, {"task": "mmlu_professional_psychology:mc", "acc_raw": 0.24019607843137256, "acc_per_token": 0.24019607843137256, "acc_per_char": 0.24019607843137256, "correct_loss_raw": 1.4043345170862533, "incorrect_loss_raw": 1.4039143988120013, "correct_loss_per_token": 1.4043345170862533, "incorrect_loss_per_token": 1.4039143988120013, "correct_loss_per_char": 0.7021672585431267, "incorrect_loss_per_char": 0.7019571994060007, "primary_score": 0.24019607843137256, "num_instances": 612, "task_config": {"task_name": "mmlu_professional_psychology:mc", "task_core": "mmlu_professional_psychology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_psychology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_psychology:mc::olmes"}}}, {"task": "mmlu_public_relations:mc", "acc_raw": 0.24545454545454545, "acc_per_token": 0.24545454545454545, "acc_per_char": 0.24545454545454545, "correct_loss_raw": 1.3907646802338687, "incorrect_loss_raw": 1.4154231024510933, "correct_loss_per_token": 1.3907646802338687, "incorrect_loss_per_token": 1.4154231024510933, "correct_loss_per_char": 0.6953823401169343, "incorrect_loss_per_char": 0.7077115512255466, "primary_score": 0.24545454545454545, "num_instances": 110, "task_config": {"task_name": "mmlu_public_relations:mc", "task_core": "mmlu_public_relations", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "public_relations", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_public_relations:mc::olmes"}}}, {"task": "mmlu_security_studies:mc", "acc_raw": 0.1836734693877551, "acc_per_token": 0.1836734693877551, "acc_per_char": 0.1836734693877551, "correct_loss_raw": 1.4304668886320933, "incorrect_loss_raw": 1.3972869594891864, "correct_loss_per_token": 1.4304668886320933, "incorrect_loss_per_token": 1.3972869594891864, "correct_loss_per_char": 0.7152334443160466, "incorrect_loss_per_char": 0.6986434797445932, "primary_score": 0.1836734693877551, "num_instances": 245, "task_config": {"task_name": "mmlu_security_studies:mc", "task_core": "mmlu_security_studies", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "security_studies", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_security_studies:mc::olmes"}}}, {"task": "mmlu_sociology:mc", "acc_raw": 0.20398009950248755, "acc_per_token": 0.20398009950248755, "acc_per_char": 0.20398009950248755, "correct_loss_raw": 1.4055783861312108, "incorrect_loss_raw": 1.4032114724417035, "correct_loss_per_token": 1.4055783861312108, "incorrect_loss_per_token": 1.4032114724417035, "correct_loss_per_char": 0.7027891930656054, "incorrect_loss_per_char": 0.7016057362208518, "primary_score": 0.20398009950248755, "num_instances": 201, "task_config": {"task_name": "mmlu_sociology:mc", "task_core": "mmlu_sociology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "sociology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_sociology:mc::olmes"}}}, {"task": "mmlu_us_foreign_policy:mc", "acc_raw": 0.25, "acc_per_token": 0.25, "acc_per_char": 0.25, "correct_loss_raw": 1.3994287383556365, "incorrect_loss_raw": 1.401831651926041, "correct_loss_per_token": 1.3994287383556365, "incorrect_loss_per_token": 1.401831651926041, "correct_loss_per_char": 0.6997143691778183, "incorrect_loss_per_char": 0.7009158259630205, "primary_score": 0.25, "num_instances": 100, "task_config": {"task_name": "mmlu_us_foreign_policy:mc", "task_core": "mmlu_us_foreign_policy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "us_foreign_policy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_us_foreign_policy:mc::olmes"}}}, {"task": "mmlu_virology:mc", "acc_raw": 0.27710843373493976, "acc_per_token": 0.27710843373493976, "acc_per_char": 0.27710843373493976, "correct_loss_raw": 1.3888891016144351, "incorrect_loss_raw": 1.4161239694878751, "correct_loss_per_token": 1.3888891016144351, "incorrect_loss_per_token": 1.4161239694878751, "correct_loss_per_char": 0.6944445508072176, "incorrect_loss_per_char": 0.7080619847439376, "primary_score": 0.27710843373493976, "num_instances": 166, "task_config": {"task_name": "mmlu_virology:mc", "task_core": "mmlu_virology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "virology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_virology:mc::olmes"}}}, {"task": "mmlu_world_religions:mc", "acc_raw": 0.23976608187134502, "acc_per_token": 0.23976608187134502, "acc_per_char": 0.23976608187134502, "correct_loss_raw": 1.3958141071754588, "incorrect_loss_raw": 1.39785935144443, "correct_loss_per_token": 1.3958141071754588, "incorrect_loss_per_token": 1.39785935144443, "correct_loss_per_char": 0.6979070535877294, "incorrect_loss_per_char": 0.698929675722215, "primary_score": 0.23976608187134502, "num_instances": 171, "task_config": {"task_name": "mmlu_world_religions:mc", "task_core": "mmlu_world_religions", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_raw", "random_subsample_seed": 1234, "context_kwargs": {"no_label_prefix_space": false}, "generation_kwargs": null, "metric_kwargs": null, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "world_religions", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_world_religions:mc::olmes"}}}, {"task": "mmlu_abstract_algebra", "acc_raw": 0.22, "acc_per_token": 0.27, "acc_per_char": 0.25, "correct_loss_raw": 6.4551828312873845, "incorrect_loss_raw": 4.987657780647279, "correct_loss_per_token": 1.7790700636836112, "incorrect_loss_per_token": 1.9649607666646502, "correct_loss_per_char": 0.7085478552367768, "incorrect_loss_per_char": 0.7159901483865085, "acc_uncond": 0.3, "correct_loss_uncond": -9.357519872188568, "incorrect_loss_uncond": -8.862104341189067, "primary_score": 0.25, "num_instances": 100, "task_config": {"task_name": "mmlu_abstract_algebra", "task_core": "mmlu_abstract_algebra", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_abstract_algebra:rc::olmes"}}}, {"task": "mmlu_anatomy", "acc_raw": 0.34074074074074073, "acc_per_token": 0.2962962962962963, "acc_per_char": 0.34074074074074073, "correct_loss_raw": 17.91354572199009, "incorrect_loss_raw": 18.324018736238834, "correct_loss_per_token": 2.270532707861062, "incorrect_loss_per_token": 2.6184480049899297, "correct_loss_per_char": 0.5162493756103963, "incorrect_loss_per_char": 0.5941078473379293, "acc_uncond": 0.31851851851851853, "correct_loss_uncond": -14.984284937381744, "incorrect_loss_uncond": -14.478863314640376, "primary_score": 0.34074074074074073, "num_instances": 135, "task_config": {"task_name": "mmlu_anatomy", "task_core": "mmlu_anatomy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "anatomy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_anatomy:rc::olmes"}}}, {"task": "mmlu_astronomy", "acc_raw": 0.3223684210526316, "acc_per_token": 0.39473684210526316, "acc_per_char": 0.4144736842105263, "correct_loss_raw": 25.177843918925838, "incorrect_loss_raw": 25.155770634872887, "correct_loss_per_token": 2.324497508223701, "incorrect_loss_per_token": 2.811709351684318, "correct_loss_per_char": 0.5685864552887461, "incorrect_loss_per_char": 0.6720247721294323, "acc_uncond": 0.42105263157894735, "correct_loss_uncond": -15.207309873480545, "incorrect_loss_uncond": -14.05902164703921, "primary_score": 0.4144736842105263, "num_instances": 152, "task_config": {"task_name": "mmlu_astronomy", "task_core": "mmlu_astronomy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "astronomy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_astronomy:rc::olmes"}}}, {"task": "mmlu_business_ethics", "acc_raw": 0.53, "acc_per_token": 0.45, "acc_per_char": 0.46, "correct_loss_raw": 22.757138235569002, "incorrect_loss_raw": 25.77794682184854, "correct_loss_per_token": 3.1831077655639537, "incorrect_loss_per_token": 3.552855390475315, "correct_loss_per_char": 0.8752598337381101, "incorrect_loss_per_char": 0.9435633335962644, "acc_uncond": 0.37, "correct_loss_uncond": -12.232631995677949, "incorrect_loss_uncond": -11.214106442133586, "primary_score": 0.46, "num_instances": 100, "task_config": {"task_name": "mmlu_business_ethics", "task_core": "mmlu_business_ethics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "business_ethics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_business_ethics:rc::olmes"}}}, {"task": "mmlu_clinical_knowledge", "acc_raw": 0.2792452830188679, "acc_per_token": 0.3433962264150943, "acc_per_char": 0.3660377358490566, "correct_loss_raw": 22.166429183618078, "incorrect_loss_raw": 20.489821842631446, "correct_loss_per_token": 2.5494427778993702, "incorrect_loss_per_token": 2.8195764623307342, "correct_loss_per_char": 0.6004814047788665, "incorrect_loss_per_char": 0.6851914042506143, "acc_uncond": 0.33962264150943394, "correct_loss_uncond": -13.612649080438434, "incorrect_loss_uncond": -12.529704328303072, "primary_score": 0.3660377358490566, "num_instances": 265, "task_config": {"task_name": "mmlu_clinical_knowledge", "task_core": "mmlu_clinical_knowledge", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "clinical_knowledge", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_clinical_knowledge:rc::olmes"}}}, {"task": "mmlu_college_biology", "acc_raw": 0.3680555555555556, "acc_per_token": 0.3611111111111111, "acc_per_char": 0.3819444444444444, "correct_loss_raw": 19.9105933378968, "incorrect_loss_raw": 21.72963875973667, "correct_loss_per_token": 2.5496391006038817, "incorrect_loss_per_token": 2.985122184704055, "correct_loss_per_char": 0.4947993088349139, "incorrect_loss_per_char": 0.5861087337984283, "acc_uncond": 0.3472222222222222, "correct_loss_uncond": -15.96084564882848, "incorrect_loss_uncond": -14.809113546654025, "primary_score": 0.3819444444444444, "num_instances": 144, "task_config": {"task_name": "mmlu_college_biology", "task_core": "mmlu_college_biology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_biology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_biology:rc::olmes"}}}, {"task": "mmlu_college_chemistry", "acc_raw": 0.22, "acc_per_token": 0.28, "acc_per_char": 0.26, "correct_loss_raw": 18.67529359340668, "incorrect_loss_raw": 18.623361633221297, "correct_loss_per_token": 2.913132373962731, "incorrect_loss_per_token": 2.9985381063616385, "correct_loss_per_char": 1.1796746788592554, "incorrect_loss_per_char": 1.1639814629662026, "acc_uncond": 0.26, "correct_loss_uncond": -12.370259900093078, "incorrect_loss_uncond": -12.005320732196175, "primary_score": 0.26, "num_instances": 100, "task_config": {"task_name": "mmlu_college_chemistry", "task_core": "mmlu_college_chemistry", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_chemistry", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_chemistry:rc::olmes"}}}, {"task": "mmlu_college_computer_science", "acc_raw": 0.23, "acc_per_token": 0.24, "acc_per_char": 0.29, "correct_loss_raw": 17.125564770698546, "incorrect_loss_raw": 16.317387603123983, "correct_loss_per_token": 2.396058769024732, "incorrect_loss_per_token": 2.5570406531858145, "correct_loss_per_char": 0.7816295440981262, "incorrect_loss_per_char": 0.7795670748224239, "acc_uncond": 0.27, "correct_loss_uncond": -12.883329787254333, "incorrect_loss_uncond": -13.138652811050415, "primary_score": 0.29, "num_instances": 100, "task_config": {"task_name": "mmlu_college_computer_science", "task_core": "mmlu_college_computer_science", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_computer_science", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_computer_science:rc::olmes"}}}, {"task": "mmlu_college_mathematics", "acc_raw": 0.15, "acc_per_token": 0.22, "acc_per_char": 0.19, "correct_loss_raw": 11.40063785791397, "incorrect_loss_raw": 10.084527692000073, "correct_loss_per_token": 2.8536593351442985, "incorrect_loss_per_token": 2.8226579726819767, "correct_loss_per_char": 1.1760315024355075, "incorrect_loss_per_char": 1.1311194345135664, "acc_uncond": 0.3, "correct_loss_uncond": -9.223767383098602, "incorrect_loss_uncond": -8.866132938861842, "primary_score": 0.19, "num_instances": 100, "task_config": {"task_name": "mmlu_college_mathematics", "task_core": "mmlu_college_mathematics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_mathematics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_mathematics:rc::olmes"}}}, {"task": "mmlu_college_medicine", "acc_raw": 0.30057803468208094, "acc_per_token": 0.3063583815028902, "acc_per_char": 0.28901734104046245, "correct_loss_raw": 19.81116359357889, "incorrect_loss_raw": 19.46911873684222, "correct_loss_per_token": 2.6087844780290466, "incorrect_loss_per_token": 2.7511619527855347, "correct_loss_per_char": 0.6122792239099126, "incorrect_loss_per_char": 0.6498799604226275, "acc_uncond": 0.3352601156069364, "correct_loss_uncond": -13.740935204346055, "incorrect_loss_uncond": -13.052801668529113, "primary_score": 0.28901734104046245, "num_instances": 173, "task_config": {"task_name": "mmlu_college_medicine", "task_core": "mmlu_college_medicine", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_medicine", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_medicine:rc::olmes"}}}, {"task": "mmlu_college_physics", "acc_raw": 0.24509803921568626, "acc_per_token": 0.2647058823529412, "acc_per_char": 0.24509803921568626, "correct_loss_raw": 12.496067006213993, "incorrect_loss_raw": 10.78362273858264, "correct_loss_per_token": 2.6446954790713253, "incorrect_loss_per_token": 2.480259769965165, "correct_loss_per_char": 1.0753204948654511, "incorrect_loss_per_char": 0.9993043140143509, "acc_uncond": 0.2549019607843137, "correct_loss_uncond": -11.738114963559543, "incorrect_loss_uncond": -11.791085632797936, "primary_score": 0.24509803921568626, "num_instances": 102, "task_config": {"task_name": "mmlu_college_physics", "task_core": "mmlu_college_physics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "college_physics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_college_physics:rc::olmes"}}}, {"task": "mmlu_computer_security", "acc_raw": 0.44, "acc_per_token": 0.39, "acc_per_char": 0.43, "correct_loss_raw": 21.590673285126687, "incorrect_loss_raw": 21.24347382863363, "correct_loss_per_token": 3.266507198900353, "incorrect_loss_per_token": 3.9848627693716905, "correct_loss_per_char": 0.7648403250653502, "incorrect_loss_per_char": 0.9002014855324758, "acc_uncond": 0.48, "correct_loss_uncond": -11.743729857802391, "incorrect_loss_uncond": -9.172414719263715, "primary_score": 0.43, "num_instances": 100, "task_config": {"task_name": "mmlu_computer_security", "task_core": "mmlu_computer_security", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "computer_security", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_computer_security:rc::olmes"}}}, {"task": "mmlu_conceptual_physics", "acc_raw": 0.4297872340425532, "acc_per_token": 0.3872340425531915, "acc_per_char": 0.37872340425531914, "correct_loss_raw": 8.800380459364424, "incorrect_loss_raw": 10.790937607085455, "correct_loss_per_token": 2.8105938294911996, "incorrect_loss_per_token": 3.5675698091199823, "correct_loss_per_char": 0.5748582898927789, "incorrect_loss_per_char": 0.6984745809759881, "acc_uncond": 0.35319148936170214, "correct_loss_uncond": -11.130239910648225, "incorrect_loss_uncond": -9.603436176404886, "primary_score": 0.37872340425531914, "num_instances": 235, "task_config": {"task_name": "mmlu_conceptual_physics", "task_core": "mmlu_conceptual_physics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "conceptual_physics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_conceptual_physics:rc::olmes"}}}, {"task": "mmlu_econometrics", "acc_raw": 0.2719298245614035, "acc_per_token": 0.2894736842105263, "acc_per_char": 0.2807017543859649, "correct_loss_raw": 20.503092958216083, "incorrect_loss_raw": 21.490892941840208, "correct_loss_per_token": 2.2654041138946712, "incorrect_loss_per_token": 2.2292898427550645, "correct_loss_per_char": 0.5412424556774481, "incorrect_loss_per_char": 0.557237568210862, "acc_uncond": 0.2894736842105263, "correct_loss_uncond": -15.166060527165731, "incorrect_loss_uncond": -14.935436847962826, "primary_score": 0.2807017543859649, "num_instances": 114, "task_config": {"task_name": "mmlu_econometrics", "task_core": "mmlu_econometrics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "econometrics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_econometrics:rc::olmes"}}}, {"task": "mmlu_electrical_engineering", "acc_raw": 0.25517241379310346, "acc_per_token": 0.31724137931034485, "acc_per_char": 0.2482758620689655, "correct_loss_raw": 13.839519211341594, "incorrect_loss_raw": 13.613122390056475, "correct_loss_per_token": 3.3820394688821844, "incorrect_loss_per_token": 3.688959328273953, "correct_loss_per_char": 0.9524712168695203, "incorrect_loss_per_char": 0.9578910470850094, "acc_uncond": 0.23448275862068965, "correct_loss_uncond": -8.453801628638958, "incorrect_loss_uncond": -9.51819067960498, "primary_score": 0.2482758620689655, "num_instances": 145, "task_config": {"task_name": "mmlu_electrical_engineering", "task_core": "mmlu_electrical_engineering", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "electrical_engineering", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_electrical_engineering:rc::olmes"}}}, {"task": "mmlu_elementary_mathematics", "acc_raw": 0.24338624338624337, "acc_per_token": 0.24338624338624337, "acc_per_char": 0.2751322751322751, "correct_loss_raw": 11.930546489657548, "incorrect_loss_raw": 12.10207574045847, "correct_loss_per_token": 3.8344038402479708, "incorrect_loss_per_token": 3.9704004893855487, "correct_loss_per_char": 1.5278709612794505, "incorrect_loss_per_char": 1.5513099371206358, "acc_uncond": 0.26455026455026454, "correct_loss_uncond": -8.801304420781515, "incorrect_loss_uncond": -8.569474821473346, "primary_score": 0.2751322751322751, "num_instances": 378, "task_config": {"task_name": "mmlu_elementary_mathematics", "task_core": "mmlu_elementary_mathematics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "elementary_mathematics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_elementary_mathematics:rc::olmes"}}}, {"task": "mmlu_formal_logic", "acc_raw": 0.30952380952380953, "acc_per_token": 0.2857142857142857, "acc_per_char": 0.29365079365079366, "correct_loss_raw": 23.540814636245607, "incorrect_loss_raw": 24.555815297459816, "correct_loss_per_token": 2.5186289372363673, "incorrect_loss_per_token": 2.5082171115158514, "correct_loss_per_char": 1.1576052453427086, "incorrect_loss_per_char": 1.1826633041442995, "acc_uncond": 0.2619047619047619, "correct_loss_uncond": -25.623597778971234, "incorrect_loss_uncond": -25.936248189557798, "primary_score": 0.29365079365079366, "num_instances": 126, "task_config": {"task_name": "mmlu_formal_logic", "task_core": "mmlu_formal_logic", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "formal_logic", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_formal_logic:rc::olmes"}}}, {"task": "mmlu_global_facts", "acc_raw": 0.29, "acc_per_token": 0.27, "acc_per_char": 0.29, "correct_loss_raw": 7.879513807296753, "incorrect_loss_raw": 8.528802576065067, "correct_loss_per_token": 2.7290880534026933, "incorrect_loss_per_token": 2.750701302564733, "correct_loss_per_char": 1.0806024398000555, "incorrect_loss_per_char": 1.1004074488770166, "acc_uncond": 0.28, "correct_loss_uncond": -7.5427256774902345, "incorrect_loss_uncond": -7.546341579755147, "primary_score": 0.29, "num_instances": 100, "task_config": {"task_name": "mmlu_global_facts", "task_core": "mmlu_global_facts", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "global_facts", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_global_facts:rc::olmes"}}}, {"task": "mmlu_high_school_biology", "acc_raw": 0.34516129032258064, "acc_per_token": 0.3870967741935484, "acc_per_char": 0.3903225806451613, "correct_loss_raw": 21.363152931582544, "incorrect_loss_raw": 21.78373695650408, "correct_loss_per_token": 2.5451589527874114, "incorrect_loss_per_token": 2.929591839395843, "correct_loss_per_char": 0.5453848861018671, "incorrect_loss_per_char": 0.5849483112641993, "acc_uncond": 0.3774193548387097, "correct_loss_uncond": -14.032325724632509, "incorrect_loss_uncond": -12.629724754312988, "primary_score": 0.3903225806451613, "num_instances": 310, "task_config": {"task_name": "mmlu_high_school_biology", "task_core": "mmlu_high_school_biology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_biology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_biology:rc::olmes"}}}, {"task": "mmlu_high_school_chemistry", "acc_raw": 0.19704433497536947, "acc_per_token": 0.2315270935960591, "acc_per_char": 0.21674876847290642, "correct_loss_raw": 20.89802763467939, "incorrect_loss_raw": 18.928701400561092, "correct_loss_per_token": 2.7035825977303256, "incorrect_loss_per_token": 2.617592187327384, "correct_loss_per_char": 0.9237622521546132, "incorrect_loss_per_char": 0.9029845173498553, "acc_uncond": 0.28078817733990147, "correct_loss_uncond": -13.191290159237209, "incorrect_loss_uncond": -13.121985284565705, "primary_score": 0.21674876847290642, "num_instances": 203, "task_config": {"task_name": "mmlu_high_school_chemistry", "task_core": "mmlu_high_school_chemistry", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_chemistry", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_chemistry:rc::olmes"}}}, {"task": "mmlu_high_school_computer_science", "acc_raw": 0.24, "acc_per_token": 0.25, "acc_per_char": 0.3, "correct_loss_raw": 23.00562346100807, "incorrect_loss_raw": 23.226041058301917, "correct_loss_per_token": 2.5115057369712956, "incorrect_loss_per_token": 2.739964992086951, "correct_loss_per_char": 0.7942636222370791, "incorrect_loss_per_char": 0.8636869777631533, "acc_uncond": 0.32, "correct_loss_uncond": -15.524907387495041, "incorrect_loss_uncond": -15.119951387643816, "primary_score": 0.3, "num_instances": 100, "task_config": {"task_name": "mmlu_high_school_computer_science", "task_core": "mmlu_high_school_computer_science", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_computer_science", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_computer_science:rc::olmes"}}}, {"task": "mmlu_high_school_european_history", "acc_raw": 0.3515151515151515, "acc_per_token": 0.45454545454545453, "acc_per_char": 0.4484848484848485, "correct_loss_raw": 27.50270097797567, "incorrect_loss_raw": 27.05405940070296, "correct_loss_per_token": 2.4626567790590443, "incorrect_loss_per_token": 3.073838508885275, "correct_loss_per_char": 0.44729345815871113, "incorrect_loss_per_char": 0.552629013230489, "acc_uncond": 0.4, "correct_loss_uncond": -15.371065504984422, "incorrect_loss_uncond": -13.31542071308752, "primary_score": 0.4484848484848485, "num_instances": 165, "task_config": {"task_name": "mmlu_high_school_european_history", "task_core": "mmlu_high_school_european_history", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_european_history", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_european_history:rc::olmes"}}}, {"task": "mmlu_high_school_geography", "acc_raw": 0.3838383838383838, "acc_per_token": 0.4292929292929293, "acc_per_char": 0.42424242424242425, "correct_loss_raw": 14.490383876694573, "incorrect_loss_raw": 14.81266781717839, "correct_loss_per_token": 3.1006659141581205, "incorrect_loss_per_token": 3.6240699168801833, "correct_loss_per_char": 0.5874396804149111, "incorrect_loss_per_char": 0.7109279375675324, "acc_uncond": 0.41414141414141414, "correct_loss_uncond": -10.85394283135732, "incorrect_loss_uncond": -9.224186990999613, "primary_score": 0.42424242424242425, "num_instances": 198, "task_config": {"task_name": "mmlu_high_school_geography", "task_core": "mmlu_high_school_geography", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_geography", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_geography:rc::olmes"}}}, {"task": "mmlu_high_school_government_and_politics", "acc_raw": 0.39896373056994816, "acc_per_token": 0.47150259067357514, "acc_per_char": 0.45077720207253885, "correct_loss_raw": 21.419602970655408, "incorrect_loss_raw": 23.24519727143599, "correct_loss_per_token": 2.236870015309941, "incorrect_loss_per_token": 2.7710672404875227, "correct_loss_per_char": 0.3740266779895104, "incorrect_loss_per_char": 0.46483421379674705, "acc_uncond": 0.45595854922279794, "correct_loss_uncond": -16.38661623900573, "incorrect_loss_uncond": -13.596990226258063, "primary_score": 0.45077720207253885, "num_instances": 193, "task_config": {"task_name": "mmlu_high_school_government_and_politics", "task_core": "mmlu_high_school_government_and_politics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_government_and_politics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_government_and_politics:rc::olmes"}}}, {"task": "mmlu_high_school_macroeconomics", "acc_raw": 0.29743589743589743, "acc_per_token": 0.33589743589743587, "acc_per_char": 0.3153846153846154, "correct_loss_raw": 21.07840545391425, "incorrect_loss_raw": 21.375357272074773, "correct_loss_per_token": 2.608882898964855, "incorrect_loss_per_token": 2.824826016663295, "correct_loss_per_char": 0.562448383317162, "incorrect_loss_per_char": 0.5901391587490764, "acc_uncond": 0.3384615384615385, "correct_loss_uncond": -15.114438359859662, "incorrect_loss_uncond": -14.110496433372163, "primary_score": 0.3153846153846154, "num_instances": 390, "task_config": {"task_name": "mmlu_high_school_macroeconomics", "task_core": "mmlu_high_school_macroeconomics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_macroeconomics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_macroeconomics:rc::olmes"}}}, {"task": "mmlu_high_school_mathematics", "acc_raw": 0.15925925925925927, "acc_per_token": 0.17777777777777778, "acc_per_char": 0.16296296296296298, "correct_loss_raw": 9.28662220283791, "incorrect_loss_raw": 8.163666827148862, "correct_loss_per_token": 4.178258001474584, "incorrect_loss_per_token": 3.917127378658242, "correct_loss_per_char": 1.7174042732985988, "incorrect_loss_per_char": 1.616340709285345, "acc_uncond": 0.2518518518518518, "correct_loss_uncond": -5.568703559592918, "incorrect_loss_uncond": -5.421064988919243, "primary_score": 0.16296296296296298, "num_instances": 270, "task_config": {"task_name": "mmlu_high_school_mathematics", "task_core": "mmlu_high_school_mathematics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_mathematics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_mathematics:rc::olmes"}}}, {"task": "mmlu_high_school_microeconomics", "acc_raw": 0.31092436974789917, "acc_per_token": 0.35294117647058826, "acc_per_char": 0.36134453781512604, "correct_loss_raw": 24.63945012733716, "incorrect_loss_raw": 24.164019934436524, "correct_loss_per_token": 2.602873918455077, "incorrect_loss_per_token": 2.8529698773609264, "correct_loss_per_char": 0.5514129796237556, "incorrect_loss_per_char": 0.5945063509843619, "acc_uncond": 0.29411764705882354, "correct_loss_uncond": -15.441406973269807, "incorrect_loss_uncond": -14.559387908429335, "primary_score": 0.36134453781512604, "num_instances": 238, "task_config": {"task_name": "mmlu_high_school_microeconomics", "task_core": "mmlu_high_school_microeconomics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_microeconomics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_microeconomics:rc::olmes"}}}, {"task": "mmlu_high_school_physics", "acc_raw": 0.25165562913907286, "acc_per_token": 0.23178807947019867, "acc_per_char": 0.24503311258278146, "correct_loss_raw": 21.13985337484751, "incorrect_loss_raw": 20.35369520997897, "correct_loss_per_token": 2.4328990738547174, "incorrect_loss_per_token": 2.3560301577896197, "correct_loss_per_char": 0.8585223176217057, "incorrect_loss_per_char": 0.8402890421816588, "acc_uncond": 0.2781456953642384, "correct_loss_uncond": -15.777207565623403, "incorrect_loss_uncond": -16.102421425299404, "primary_score": 0.24503311258278146, "num_instances": 151, "task_config": {"task_name": "mmlu_high_school_physics", "task_core": "mmlu_high_school_physics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_physics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_physics:rc::olmes"}}}, {"task": "mmlu_high_school_psychology", "acc_raw": 0.48807339449541287, "acc_per_token": 0.47706422018348627, "acc_per_char": 0.47522935779816516, "correct_loss_raw": 14.156356584588321, "incorrect_loss_raw": 16.74465744896401, "correct_loss_per_token": 2.888265642439932, "incorrect_loss_per_token": 3.7921417257225736, "correct_loss_per_char": 0.48106282032416087, "incorrect_loss_per_char": 0.6332678164009538, "acc_uncond": 0.47155963302752296, "correct_loss_uncond": -13.520115094994186, "incorrect_loss_uncond": -11.29164081728057, "primary_score": 0.47522935779816516, "num_instances": 545, "task_config": {"task_name": "mmlu_high_school_psychology", "task_core": "mmlu_high_school_psychology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_psychology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_psychology:rc::olmes"}}}, {"task": "mmlu_high_school_statistics", "acc_raw": 0.27314814814814814, "acc_per_token": 0.2916666666666667, "acc_per_char": 0.2777777777777778, "correct_loss_raw": 26.131407388382488, "incorrect_loss_raw": 26.658040807938868, "correct_loss_per_token": 2.5982270382700445, "incorrect_loss_per_token": 2.6451773859738483, "correct_loss_per_char": 0.786524301711884, "incorrect_loss_per_char": 0.8176619046460267, "acc_uncond": 0.3055555555555556, "correct_loss_uncond": -17.335551079224658, "incorrect_loss_uncond": -16.501133336329172, "primary_score": 0.2777777777777778, "num_instances": 216, "task_config": {"task_name": "mmlu_high_school_statistics", "task_core": "mmlu_high_school_statistics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_statistics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_statistics:rc::olmes"}}}, {"task": "mmlu_high_school_us_history", "acc_raw": 0.3284313725490196, "acc_per_token": 0.3431372549019608, "acc_per_char": 0.37745098039215685, "correct_loss_raw": 25.274763485672427, "incorrect_loss_raw": 26.57934519395329, "correct_loss_per_token": 2.455718281654319, "incorrect_loss_per_token": 2.757877330223681, "correct_loss_per_char": 0.4598392848516849, "incorrect_loss_per_char": 0.5243689544489788, "acc_uncond": 0.4264705882352941, "correct_loss_uncond": -13.92219268603652, "incorrect_loss_uncond": -11.630784141861536, "primary_score": 0.37745098039215685, "num_instances": 204, "task_config": {"task_name": "mmlu_high_school_us_history", "task_core": "mmlu_high_school_us_history", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_us_history", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_us_history:rc::olmes"}}}, {"task": "mmlu_high_school_world_history", "acc_raw": 0.3206751054852321, "acc_per_token": 0.37130801687763715, "acc_per_char": 0.37130801687763715, "correct_loss_raw": 29.43805429251385, "incorrect_loss_raw": 29.626125245154633, "correct_loss_per_token": 2.7216699601477736, "incorrect_loss_per_token": 3.136590812911396, "correct_loss_per_char": 0.5012342985644117, "incorrect_loss_per_char": 0.5617093694188149, "acc_uncond": 0.3924050632911392, "correct_loss_uncond": -14.541466581670544, "incorrect_loss_uncond": -12.60329805916037, "primary_score": 0.37130801687763715, "num_instances": 237, "task_config": {"task_name": "mmlu_high_school_world_history", "task_core": "mmlu_high_school_world_history", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "high_school_world_history", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_high_school_world_history:rc::olmes"}}}, {"task": "mmlu_human_aging", "acc_raw": 0.39461883408071746, "acc_per_token": 0.39461883408071746, "acc_per_char": 0.37668161434977576, "correct_loss_raw": 12.677635809231232, "incorrect_loss_raw": 15.47025020391595, "correct_loss_per_token": 3.046415314731483, "incorrect_loss_per_token": 3.5851252066525, "correct_loss_per_char": 0.5667559513387225, "incorrect_loss_per_char": 0.7035341763206064, "acc_uncond": 0.4260089686098655, "correct_loss_uncond": -10.355107204261916, "incorrect_loss_uncond": -8.973779054589693, "primary_score": 0.37668161434977576, "num_instances": 223, "task_config": {"task_name": "mmlu_human_aging", "task_core": "mmlu_human_aging", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "human_aging", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_human_aging:rc::olmes"}}}, {"task": "mmlu_human_sexuality", "acc_raw": 0.4122137404580153, "acc_per_token": 0.4198473282442748, "acc_per_char": 0.3969465648854962, "correct_loss_raw": 14.669895948799512, "incorrect_loss_raw": 16.163360671991004, "correct_loss_per_token": 3.0384140234268244, "incorrect_loss_per_token": 3.6212319350738484, "correct_loss_per_char": 0.6311531693924813, "incorrect_loss_per_char": 0.6829634331608414, "acc_uncond": 0.3282442748091603, "correct_loss_uncond": -10.704758468020053, "incorrect_loss_uncond": -11.668367686920798, "primary_score": 0.3969465648854962, "num_instances": 131, "task_config": {"task_name": "mmlu_human_sexuality", "task_core": "mmlu_human_sexuality", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "human_sexuality", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_human_sexuality:rc::olmes"}}}, {"task": "mmlu_international_law", "acc_raw": 0.17355371900826447, "acc_per_token": 0.30578512396694213, "acc_per_char": 0.256198347107438, "correct_loss_raw": 48.5408869026121, "incorrect_loss_raw": 35.460322001420444, "correct_loss_per_token": 2.4756060714120736, "incorrect_loss_per_token": 2.6451336287890466, "correct_loss_per_char": 0.453038230927997, "incorrect_loss_per_char": 0.4743127227123939, "acc_uncond": 0.371900826446281, "correct_loss_uncond": -25.168442221712475, "incorrect_loss_uncond": -22.507250311946084, "primary_score": 0.256198347107438, "num_instances": 121, "task_config": {"task_name": "mmlu_international_law", "task_core": "mmlu_international_law", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "international_law", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_international_law:rc::olmes"}}}, {"task": "mmlu_jurisprudence", "acc_raw": 0.23148148148148148, "acc_per_token": 0.26851851851851855, "acc_per_char": 0.3148148148148148, "correct_loss_raw": 26.89270097238046, "incorrect_loss_raw": 22.6379110702762, "correct_loss_per_token": 3.0341406842080265, "incorrect_loss_per_token": 3.5132505140292585, "correct_loss_per_char": 0.5889211461775832, "incorrect_loss_per_char": 0.6462895365021206, "acc_uncond": 0.3888888888888889, "correct_loss_uncond": -14.154463432453296, "incorrect_loss_uncond": -12.54157188718702, "primary_score": 0.3148148148148148, "num_instances": 108, "task_config": {"task_name": "mmlu_jurisprudence", "task_core": "mmlu_jurisprudence", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "jurisprudence", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_jurisprudence:rc::olmes"}}}, {"task": "mmlu_logical_fallacies", "acc_raw": 0.31901840490797545, "acc_per_token": 0.3312883435582822, "acc_per_char": 0.3067484662576687, "correct_loss_raw": 23.010878557076484, "incorrect_loss_raw": 22.950006944757778, "correct_loss_per_token": 3.1891809132318185, "incorrect_loss_per_token": 3.5550170354697945, "correct_loss_per_char": 0.5582601865429742, "incorrect_loss_per_char": 0.6406159326677359, "acc_uncond": 0.39263803680981596, "correct_loss_uncond": -13.189598791438378, "incorrect_loss_uncond": -11.083331307506748, "primary_score": 0.3067484662576687, "num_instances": 163, "task_config": {"task_name": "mmlu_logical_fallacies", "task_core": "mmlu_logical_fallacies", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "logical_fallacies", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_logical_fallacies:rc::olmes"}}}, {"task": "mmlu_machine_learning", "acc_raw": 0.25, "acc_per_token": 0.20535714285714285, "acc_per_char": 0.2767857142857143, "correct_loss_raw": 18.307162577552454, "incorrect_loss_raw": 18.575598238834324, "correct_loss_per_token": 3.585282597547379, "incorrect_loss_per_token": 3.5459706425587236, "correct_loss_per_char": 0.9230253304795293, "incorrect_loss_per_char": 0.9122802462172885, "acc_uncond": 0.24107142857142858, "correct_loss_uncond": -8.379302962550096, "incorrect_loss_uncond": -7.790933533438611, "primary_score": 0.2767857142857143, "num_instances": 112, "task_config": {"task_name": "mmlu_machine_learning", "task_core": "mmlu_machine_learning", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "machine_learning", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_machine_learning:rc::olmes"}}}, {"task": "mmlu_management", "acc_raw": 0.36893203883495146, "acc_per_token": 0.4368932038834951, "acc_per_char": 0.5145631067961165, "correct_loss_raw": 13.354300859700707, "incorrect_loss_raw": 14.035566178726148, "correct_loss_per_token": 3.448802221833666, "incorrect_loss_per_token": 3.9716159753555917, "correct_loss_per_char": 0.5887009092409919, "incorrect_loss_per_char": 0.6663220082502314, "acc_uncond": 0.4174757281553398, "correct_loss_uncond": -9.784205390773351, "incorrect_loss_uncond": -8.292069842514481, "primary_score": 0.5145631067961165, "num_instances": 103, "task_config": {"task_name": "mmlu_management", "task_core": "mmlu_management", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "management", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_management:rc::olmes"}}}, {"task": "mmlu_marketing", "acc_raw": 0.5213675213675214, "acc_per_token": 0.5042735042735043, "acc_per_char": 0.5427350427350427, "correct_loss_raw": 13.019673931293, "incorrect_loss_raw": 16.553809684735754, "correct_loss_per_token": 2.6529033287731636, "incorrect_loss_per_token": 3.59070879885069, "correct_loss_per_char": 0.5415795828438098, "incorrect_loss_per_char": 0.7426038180350188, "acc_uncond": 0.5299145299145299, "correct_loss_uncond": -12.220347578708942, "incorrect_loss_uncond": -9.38660909935959, "primary_score": 0.5427350427350427, "num_instances": 234, "task_config": {"task_name": "mmlu_marketing", "task_core": "mmlu_marketing", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "marketing", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_marketing:rc::olmes"}}}, {"task": "mmlu_medical_genetics", "acc_raw": 0.38, "acc_per_token": 0.42, "acc_per_char": 0.43, "correct_loss_raw": 14.743131907582283, "incorrect_loss_raw": 14.271686580975851, "correct_loss_per_token": 2.4686744897557973, "incorrect_loss_per_token": 2.9509285705860315, "correct_loss_per_char": 0.6537800260975594, "incorrect_loss_per_char": 0.7681347658972387, "acc_uncond": 0.37, "correct_loss_uncond": -13.33009413421154, "incorrect_loss_uncond": -11.562404470443727, "primary_score": 0.43, "num_instances": 100, "task_config": {"task_name": "mmlu_medical_genetics", "task_core": "mmlu_medical_genetics", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "medical_genetics", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_medical_genetics:rc::olmes"}}}, {"task": "mmlu_miscellaneous", "acc_raw": 0.4827586206896552, "acc_per_token": 0.4840357598978289, "acc_per_char": 0.48020434227330777, "correct_loss_raw": 9.973246936668495, "incorrect_loss_raw": 12.443472592563705, "correct_loss_per_token": 2.8331898448315176, "incorrect_loss_per_token": 4.036465953696992, "correct_loss_per_char": 0.6163125125320722, "incorrect_loss_per_char": 0.8791243173215028, "acc_uncond": 0.4840357598978289, "correct_loss_uncond": -10.355883860273114, "incorrect_loss_uncond": -7.90547576311955, "primary_score": 0.48020434227330777, "num_instances": 783, "task_config": {"task_name": "mmlu_miscellaneous", "task_core": "mmlu_miscellaneous", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "miscellaneous", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_miscellaneous:rc::olmes"}}}, {"task": "mmlu_moral_disputes", "acc_raw": 0.2861271676300578, "acc_per_token": 0.2832369942196532, "acc_per_char": 0.2514450867052023, "correct_loss_raw": 26.762201377249866, "incorrect_loss_raw": 24.295487502177107, "correct_loss_per_token": 2.8838279812730208, "incorrect_loss_per_token": 3.0668071172461397, "correct_loss_per_char": 0.5684422696554423, "incorrect_loss_per_char": 0.5808929020841671, "acc_uncond": 0.3063583815028902, "correct_loss_uncond": -13.697513322092894, "incorrect_loss_uncond": -13.334110145164594, "primary_score": 0.2514450867052023, "num_instances": 346, "task_config": {"task_name": "mmlu_moral_disputes", "task_core": "mmlu_moral_disputes", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "moral_disputes", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_moral_disputes:rc::olmes"}}}, {"task": "mmlu_moral_scenarios", "acc_raw": 0.23798882681564246, "acc_per_token": 0.24134078212290502, "acc_per_char": 0.24022346368715083, "correct_loss_raw": 1.7945288913210011, "incorrect_loss_raw": 1.797925722465836, "correct_loss_per_token": 0.4390254018226819, "incorrect_loss_per_token": 0.44364589957625594, "correct_loss_per_char": 0.10330200633329756, "incorrect_loss_per_char": 0.10433826118203812, "acc_uncond": 0.27262569832402234, "correct_loss_uncond": -20.481219156960535, "incorrect_loss_uncond": -20.232290562770878, "primary_score": 0.24022346368715083, "num_instances": 895, "task_config": {"task_name": "mmlu_moral_scenarios", "task_core": "mmlu_moral_scenarios", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "moral_scenarios", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_moral_scenarios:rc::olmes"}}}, {"task": "mmlu_nutrition", "acc_raw": 0.2647058823529412, "acc_per_token": 0.3137254901960784, "acc_per_char": 0.32679738562091504, "correct_loss_raw": 26.710010400589773, "incorrect_loss_raw": 23.848608736768515, "correct_loss_per_token": 2.6501611735502424, "incorrect_loss_per_token": 2.902847647897238, "correct_loss_per_char": 0.5759508009400872, "incorrect_loss_per_char": 0.6318093526945697, "acc_uncond": 0.3333333333333333, "correct_loss_uncond": -11.975636066369761, "incorrect_loss_uncond": -11.679631917954525, "primary_score": 0.32679738562091504, "num_instances": 306, "task_config": {"task_name": "mmlu_nutrition", "task_core": "mmlu_nutrition", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "nutrition", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_nutrition:rc::olmes"}}}, {"task": "mmlu_philosophy", "acc_raw": 0.2990353697749196, "acc_per_token": 0.31511254019292606, "acc_per_char": 0.3022508038585209, "correct_loss_raw": 22.036362408154265, "incorrect_loss_raw": 20.702107290363003, "correct_loss_per_token": 3.1140666275385387, "incorrect_loss_per_token": 3.35463497433172, "correct_loss_per_char": 0.6169163719681281, "incorrect_loss_per_char": 0.6499129989768088, "acc_uncond": 0.3633440514469453, "correct_loss_uncond": -13.029168019244908, "incorrect_loss_uncond": -12.086915756345185, "primary_score": 0.3022508038585209, "num_instances": 311, "task_config": {"task_name": "mmlu_philosophy", "task_core": "mmlu_philosophy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "philosophy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_philosophy:rc::olmes"}}}, {"task": "mmlu_prehistory", "acc_raw": 0.404320987654321, "acc_per_token": 0.36728395061728397, "acc_per_char": 0.3425925925925926, "correct_loss_raw": 21.146028854044868, "incorrect_loss_raw": 22.864884013746988, "correct_loss_per_token": 2.632545019073288, "incorrect_loss_per_token": 2.9579714207120884, "correct_loss_per_char": 0.6003120828855978, "incorrect_loss_per_char": 0.6647959973037921, "acc_uncond": 0.38580246913580246, "correct_loss_uncond": -14.831230773050109, "incorrect_loss_uncond": -14.000013917807197, "primary_score": 0.3425925925925926, "num_instances": 324, "task_config": {"task_name": "mmlu_prehistory", "task_core": "mmlu_prehistory", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "prehistory", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_prehistory:rc::olmes"}}}, {"task": "mmlu_professional_accounting", "acc_raw": 0.2907801418439716, "acc_per_token": 0.2695035460992908, "acc_per_char": 0.2624113475177305, "correct_loss_raw": 25.166415042911016, "incorrect_loss_raw": 25.396868566953817, "correct_loss_per_token": 2.9057916461264224, "incorrect_loss_per_token": 2.9578965180040386, "correct_loss_per_char": 0.7851858868344537, "incorrect_loss_per_char": 0.815302295690538, "acc_uncond": 0.3120567375886525, "correct_loss_uncond": -12.864156899722756, "incorrect_loss_uncond": -12.295960299089451, "primary_score": 0.2624113475177305, "num_instances": 282, "task_config": {"task_name": "mmlu_professional_accounting", "task_core": "mmlu_professional_accounting", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_accounting", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_accounting:rc::olmes"}}}, {"task": "mmlu_professional_law", "acc_raw": 0.25097783572359844, "acc_per_token": 0.25684485006518903, "acc_per_char": 0.2757496740547588, "correct_loss_raw": 41.34747552964958, "incorrect_loss_raw": 39.70492565160419, "correct_loss_per_token": 2.2535205880807645, "incorrect_loss_per_token": 2.2561388734950083, "correct_loss_per_char": 0.4481341781759717, "incorrect_loss_per_char": 0.4478595031156273, "acc_uncond": 0.2848761408083442, "correct_loss_uncond": -26.798888672439496, "incorrect_loss_uncond": -25.853866037106012, "primary_score": 0.2757496740547588, "num_instances": 1534, "task_config": {"task_name": "mmlu_professional_law", "task_core": "mmlu_professional_law", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_law", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_law:rc::olmes"}}}, {"task": "mmlu_professional_medicine", "acc_raw": 0.28308823529411764, "acc_per_token": 0.3125, "acc_per_char": 0.2977941176470588, "correct_loss_raw": 14.847494749023634, "incorrect_loss_raw": 15.73212240358779, "correct_loss_per_token": 2.668888486909859, "incorrect_loss_per_token": 2.8228289677717346, "correct_loss_per_char": 0.5463753252705837, "incorrect_loss_per_char": 0.5865243782959754, "acc_uncond": 0.3713235294117647, "correct_loss_uncond": -11.705239387557787, "incorrect_loss_uncond": -10.756183107255723, "primary_score": 0.2977941176470588, "num_instances": 272, "task_config": {"task_name": "mmlu_professional_medicine", "task_core": "mmlu_professional_medicine", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_medicine", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_medicine:rc::olmes"}}}, {"task": "mmlu_professional_psychology", "acc_raw": 0.29901960784313725, "acc_per_token": 0.33986928104575165, "acc_per_char": 0.3202614379084967, "correct_loss_raw": 25.4097203849967, "incorrect_loss_raw": 26.151661396351244, "correct_loss_per_token": 3.2427745371115555, "incorrect_loss_per_token": 3.5245931682352443, "correct_loss_per_char": 0.587391723247063, "incorrect_loss_per_char": 0.6356098136551943, "acc_uncond": 0.3104575163398693, "correct_loss_uncond": -15.352141680281147, "incorrect_loss_uncond": -14.66829736215877, "primary_score": 0.3202614379084967, "num_instances": 612, "task_config": {"task_name": "mmlu_professional_psychology", "task_core": "mmlu_professional_psychology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "professional_psychology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_professional_psychology:rc::olmes"}}}, {"task": "mmlu_public_relations", "acc_raw": 0.44545454545454544, "acc_per_token": 0.39090909090909093, "acc_per_char": 0.2909090909090909, "correct_loss_raw": 13.839453978430141, "incorrect_loss_raw": 16.5949248881051, "correct_loss_per_token": 3.9873044605859063, "incorrect_loss_per_token": 4.544503142075553, "correct_loss_per_char": 0.7093345693575296, "incorrect_loss_per_char": 0.773644059946458, "acc_uncond": 0.34545454545454546, "correct_loss_uncond": -9.499438776211305, "incorrect_loss_uncond": -8.281265567649495, "primary_score": 0.2909090909090909, "num_instances": 110, "task_config": {"task_name": "mmlu_public_relations", "task_core": "mmlu_public_relations", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "public_relations", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_public_relations:rc::olmes"}}}, {"task": "mmlu_security_studies", "acc_raw": 0.30612244897959184, "acc_per_token": 0.31020408163265306, "acc_per_char": 0.27346938775510204, "correct_loss_raw": 88.36162767215652, "incorrect_loss_raw": 98.62264835591225, "correct_loss_per_token": 3.15054932583337, "incorrect_loss_per_token": 3.0993470106979735, "correct_loss_per_char": 0.6036736416103822, "incorrect_loss_per_char": 0.5617384804719124, "acc_uncond": 0.2653061224489796, "correct_loss_uncond": -16.647231608021016, "incorrect_loss_uncond": -19.05951665372265, "primary_score": 0.27346938775510204, "num_instances": 245, "task_config": {"task_name": "mmlu_security_studies", "task_core": "mmlu_security_studies", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "security_studies", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_security_studies:rc::olmes"}}}, {"task": "mmlu_sociology", "acc_raw": 0.27860696517412936, "acc_per_token": 0.34328358208955223, "acc_per_char": 0.27860696517412936, "correct_loss_raw": 29.919023765260306, "incorrect_loss_raw": 30.6563104515645, "correct_loss_per_token": 3.2267842207330983, "incorrect_loss_per_token": 3.550673093473738, "correct_loss_per_char": 0.5550078604496379, "incorrect_loss_per_char": 0.5870597591181631, "acc_uncond": 0.43283582089552236, "correct_loss_uncond": -14.502825765467401, "incorrect_loss_uncond": -13.354575955848006, "primary_score": 0.27860696517412936, "num_instances": 201, "task_config": {"task_name": "mmlu_sociology", "task_core": "mmlu_sociology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "sociology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_sociology:rc::olmes"}}}, {"task": "mmlu_us_foreign_policy", "acc_raw": 0.37, "acc_per_token": 0.34, "acc_per_char": 0.37, "correct_loss_raw": 21.595972112417222, "incorrect_loss_raw": 20.498132687409726, "correct_loss_per_token": 2.4566456377095482, "incorrect_loss_per_token": 2.835236189122907, "correct_loss_per_char": 0.48015732518820037, "incorrect_loss_per_char": 0.5316416496468698, "acc_uncond": 0.42, "correct_loss_uncond": -13.404697128534316, "incorrect_loss_uncond": -11.991759479045868, "primary_score": 0.37, "num_instances": 100, "task_config": {"task_name": "mmlu_us_foreign_policy", "task_core": "mmlu_us_foreign_policy", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "us_foreign_policy", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_us_foreign_policy:rc::olmes"}}}, {"task": "mmlu_virology", "acc_raw": 0.23493975903614459, "acc_per_token": 0.3795180722891566, "acc_per_char": 0.30120481927710846, "correct_loss_raw": 18.997218740991798, "incorrect_loss_raw": 19.065596443821622, "correct_loss_per_token": 3.2663271122144337, "incorrect_loss_per_token": 3.717375781578531, "correct_loss_per_char": 0.6537507042217882, "incorrect_loss_per_char": 0.7220563400961199, "acc_uncond": 0.28313253012048195, "correct_loss_uncond": -10.576083596930447, "incorrect_loss_uncond": -10.448030103880724, "primary_score": 0.30120481927710846, "num_instances": 166, "task_config": {"task_name": "mmlu_virology", "task_core": "mmlu_virology", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "virology", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_virology:rc::olmes"}}}, {"task": "mmlu_world_religions", "acc_raw": 0.4678362573099415, "acc_per_token": 0.49707602339181284, "acc_per_char": 0.5029239766081871, "correct_loss_raw": 8.47612531526744, "incorrect_loss_raw": 10.434313575542925, "correct_loss_per_token": 2.4943425273595117, "incorrect_loss_per_token": 3.6110528083283655, "correct_loss_per_char": 0.6646023540305831, "incorrect_loss_per_char": 0.9210858793720432, "acc_uncond": 0.5380116959064327, "correct_loss_uncond": -10.584433940767545, "incorrect_loss_uncond": -7.8370108089716535, "primary_score": 0.5029239766081871, "num_instances": 171, "task_config": {"task_name": "mmlu_world_religions", "task_core": "mmlu_world_religions", "limit": null, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000}, "native_id_field": "index", "fewshot_source": null, "dataset_path": "cais/mmlu", "dataset_name": "world_religions", "use_chat_format": null, "version": 1, "revision": null, "metadata": {"regimes": ["OLMES-v0.1"], "alias": "mmlu_world_religions:rc::olmes"}}}], "model_config": {"model": "hf_checkpoints/dclm-pool-1b-1x-h-fasttext-open_lm_1b_swiglutorch-warm5000-lr0p003-wd0p033-cd3e-05-bs256-mult1-seed124-tokens28795904000"}}