Add eval

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_2.csv

@@ -12,6 +12,10 @@ cb,f1,0.2557471264367816,,1
 copa,acc,0.73,0.044619604333847394,0
 hellaswag,acc,0.45030870344552876,0.004965078477435579,0
 hellaswag,acc_norm,0.599183429595698,0.004890623693243619,0
+piqa,acc,0.7524483133841132,0.010069703966857106,0
+piqa,acc_norm,0.750272034820457,0.010099232969867469,0
 rte,acc,0.5342960288808665,0.030025579819366426,0
+sciq,acc,0.881,0.010244215145336664,0
+sciq,acc_norm,0.87,0.010640169792499349,0
 storycloze_2016,acc,0.6996258685195083,0.010600915927985021,0
 winogrande,acc,0.5611681136543015,0.013946933444507032,0

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_2.json

@@ -54,6 +54,18 @@
             "acc_stderr": 0.012980954547659554,
             "acc_norm": 0.2883959044368601,
             "acc_norm_stderr": 0.013238394422428175
+        },
+        "sciq": {
+            "acc": 0.881,
+            "acc_stderr": 0.010244215145336664,
+            "acc_norm": 0.87,
+            "acc_norm_stderr": 0.010640169792499349
+        },
+        "piqa": {
+            "acc": 0.7524483133841132,
+            "acc_stderr": 0.010069703966857106,
+            "acc_norm": 0.750272034820457,
+            "acc_norm_stderr": 0.010099232969867469
         }
     },
     "versions": {
@@ -68,6 +80,8 @@
         "storycloze_2016": 0,
         "boolq": 1,
         "arc_easy": 0,
-        "arc_challenge": 0
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
     }
 }

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_3.csv

@@ -2,11 +2,20 @@ task,metric,value,err,version
 anli_r1,acc,0.33,0.014876872027456732,0
 anli_r2,acc,0.371,0.015283736211823188,0
 anli_r3,acc,0.32166666666666666,0.013490095282989521,0
+arc_challenge,acc,0.2713310580204778,0.012993807727545803,0
+arc_challenge,acc_norm,0.28498293515358364,0.013191348179838795,0
+arc_easy,acc,0.5942760942760943,0.010075755540128871,0
+arc_easy,acc_norm,0.5820707070707071,0.010120628211017883,0
+boolq,acc,0.5804281345565749,0.008631175489166717,1
 cb,acc,0.35714285714285715,0.06460957383809221,1
 cb,f1,0.3051529790660225,,1
 copa,acc,0.81,0.03942772444036623,0
 hellaswag,acc,0.45309699263095,0.004967778940011933,0
 hellaswag,acc_norm,0.6016729735112527,0.004885529674958343,0
+piqa,acc,0.750272034820457,0.010099232969867493,0
+piqa,acc_norm,0.7546245919477693,0.010039831320422387,0
 rte,acc,0.5234657039711191,0.03006330041190266,0
+sciq,acc,0.875,0.010463483381956722,0
+sciq,acc_norm,0.865,0.010811655372416051,0
 storycloze_2016,acc,0.7076429716729022,0.010518239729787736,0
 winogrande,acc,0.5706393054459353,0.01391153749996916,0

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_3.json

@@ -38,6 +38,34 @@
         "storycloze_2016": {
             "acc": 0.7076429716729022,
             "acc_stderr": 0.010518239729787736
+        },
+        "boolq": {
+            "acc": 0.5804281345565749,
+            "acc_stderr": 0.008631175489166717
+        },
+        "arc_easy": {
+            "acc": 0.5942760942760943,
+            "acc_stderr": 0.010075755540128871,
+            "acc_norm": 0.5820707070707071,
+            "acc_norm_stderr": 0.010120628211017883
+        },
+        "arc_challenge": {
+            "acc": 0.2713310580204778,
+            "acc_stderr": 0.012993807727545803,
+            "acc_norm": 0.28498293515358364,
+            "acc_norm_stderr": 0.013191348179838795
+        },
+        "sciq": {
+            "acc": 0.875,
+            "acc_stderr": 0.010463483381956722,
+            "acc_norm": 0.865,
+            "acc_norm_stderr": 0.010811655372416051
+        },
+        "piqa": {
+            "acc": 0.750272034820457,
+            "acc_stderr": 0.010099232969867493,
+            "acc_norm": 0.7546245919477693,
+            "acc_norm_stderr": 0.010039831320422387
         }
     },
     "versions": {
@@ -49,6 +77,11 @@
         "hellaswag": 0,
         "rte": 0,
         "winogrande": 0,
-        "storycloze_2016": 0
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
     }
 }

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_4.csv

@@ -2,10 +2,20 @@ task,metric,value,err,version
 anli_r1,acc,0.339,0.01497675877162034,0
 anli_r2,acc,0.342,0.01500870618212173,0
 anli_r3,acc,0.33416666666666667,0.01362243481313678,0
+arc_challenge,acc,0.2773037542662116,0.013082095839059374,0
+arc_challenge,acc_norm,0.2883959044368601,0.013238394422428173,0
+arc_easy,acc,0.6056397306397306,0.010028176038392999,0
+arc_easy,acc_norm,0.5862794612794613,0.010105878530238137,0
+boolq,acc,0.5954128440366973,0.008584355308932694,1
 cb,acc,0.375,0.06527912098338669,1
 cb,f1,0.3162533392229865,,1
 copa,acc,0.78,0.04163331998932261,0
 hellaswag,acc,0.44981079466241786,0.004964579685712441,0
 hellaswag,acc_norm,0.6027683728340968,0.0048832465794966485,0
+piqa,acc,0.7470076169749728,0.010142888698862462,0
+piqa,acc_norm,0.7540805223068553,0.010047331865625182,0
 rte,acc,0.516245487364621,0.030080573208738064,0
+sciq,acc,0.888,0.009977753031397236,0
+sciq,acc_norm,0.884,0.010131468138756976,0
+storycloze_2016,acc,0.7071084981293426,0.010523873293246304,0
 winogrande,acc,0.5816890292028414,0.013863669961195911,0

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_4.json

@@ -34,6 +34,38 @@
         "winogrande": {
             "acc": 0.5816890292028414,
             "acc_stderr": 0.013863669961195911
+        },
+        "storycloze_2016": {
+            "acc": 0.7071084981293426,
+            "acc_stderr": 0.010523873293246304
+        },
+        "boolq": {
+            "acc": 0.5954128440366973,
+            "acc_stderr": 0.008584355308932694
+        },
+        "arc_easy": {
+            "acc": 0.6056397306397306,
+            "acc_stderr": 0.010028176038392999,
+            "acc_norm": 0.5862794612794613,
+            "acc_norm_stderr": 0.010105878530238137
+        },
+        "arc_challenge": {
+            "acc": 0.2773037542662116,
+            "acc_stderr": 0.013082095839059374,
+            "acc_norm": 0.2883959044368601,
+            "acc_norm_stderr": 0.013238394422428173
+        },
+        "sciq": {
+            "acc": 0.888,
+            "acc_stderr": 0.009977753031397236,
+            "acc_norm": 0.884,
+            "acc_norm_stderr": 0.010131468138756976
+        },
+        "piqa": {
+            "acc": 0.7470076169749728,
+            "acc_stderr": 0.010142888698862462,
+            "acc_norm": 0.7540805223068553,
+            "acc_norm_stderr": 0.010047331865625182
         }
     },
     "versions": {
@@ -44,6 +76,12 @@
         "copa": 0,
         "hellaswag": 0,
         "rte": 0,
-        "winogrande": 0
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
     }
 }

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_5.csv

@@ -2,6 +2,20 @@ task,metric,value,err,version
 anli_r1,acc,0.35,0.015090650341444233,0
 anli_r2,acc,0.361,0.015195720118175127,0
 anli_r3,acc,0.31416666666666665,0.013405399314984101,0
+arc_challenge,acc,0.2764505119453925,0.013069662474252425,0
+arc_challenge,acc_norm,0.2977815699658703,0.013363080107244487,0
+arc_easy,acc,0.5942760942760943,0.010075755540128873,0
+arc_easy,acc_norm,0.5728114478114478,0.010150415974210875,0
+boolq,acc,0.5929663608562691,0.00859256288706887,1
 cb,acc,0.39285714285714285,0.0658538889806635,1
 cb,f1,0.3300527326188914,,1
 copa,acc,0.7,0.046056618647183814,0
+hellaswag,acc,0.45180242979486157,0.004966544724452228,0
+hellaswag,acc_norm,0.6045608444532962,0.00487945547466382,0
+piqa,acc,0.7453754080522307,0.01016443223706049,0
+piqa,acc_norm,0.7551686615886833,0.01003230910556881,0
+rte,acc,0.5342960288808665,0.030025579819366426,0
+sciq,acc,0.898,0.009575368801653897,0
+sciq,acc_norm,0.892,0.009820001651345703,0
+storycloze_2016,acc,0.6985569214323891,0.010611646032767588,0
+winogrande,acc,0.5603788476716653,0.013949649776015694,0

evaluation/rankeval/lm1-4b2-84b-c4-perplexity_5.json

@@ -20,6 +20,52 @@
         "copa": {
             "acc": 0.7,
             "acc_stderr": 0.046056618647183814
+        },
+        "hellaswag": {
+            "acc": 0.45180242979486157,
+            "acc_stderr": 0.004966544724452228,
+            "acc_norm": 0.6045608444532962,
+            "acc_norm_stderr": 0.00487945547466382
+        },
+        "rte": {
+            "acc": 0.5342960288808665,
+            "acc_stderr": 0.030025579819366426
+        },
+        "winogrande": {
+            "acc": 0.5603788476716653,
+            "acc_stderr": 0.013949649776015694
+        },
+        "storycloze_2016": {
+            "acc": 0.6985569214323891,
+            "acc_stderr": 0.010611646032767588
+        },
+        "boolq": {
+            "acc": 0.5929663608562691,
+            "acc_stderr": 0.00859256288706887
+        },
+        "arc_easy": {
+            "acc": 0.5942760942760943,
+            "acc_stderr": 0.010075755540128873,
+            "acc_norm": 0.5728114478114478,
+            "acc_norm_stderr": 0.010150415974210875
+        },
+        "arc_challenge": {
+            "acc": 0.2764505119453925,
+            "acc_stderr": 0.013069662474252425,
+            "acc_norm": 0.2977815699658703,
+            "acc_norm_stderr": 0.013363080107244487
+        },
+        "sciq": {
+            "acc": 0.898,
+            "acc_stderr": 0.009575368801653897,
+            "acc_norm": 0.892,
+            "acc_norm_stderr": 0.009820001651345703
+        },
+        "piqa": {
+            "acc": 0.7453754080522307,
+            "acc_stderr": 0.01016443223706049,
+            "acc_norm": 0.7551686615886833,
+            "acc_norm_stderr": 0.01003230910556881
         }
     },
     "versions": {
@@ -27,6 +73,15 @@
         "anli_r2": 0,
         "anli_r3": 0,
         "cb": 1,
-        "copa": 0
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
     }
 }