| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 12.50000000 | |
| 9 11.11111111 | |
| 10 10.00000000 | |
| 11 9.09090909 | |
| 12 16.66666667 | |
| 13 23.07692308 | |
| 14 21.42857143 | |
| 15 20.00000000 | |
| 16 18.75000000 | |
| 17 17.64705882 | |
| 18 16.66666667 | |
| 19 21.05263158 | |
| 20 20.00000000 | |
| 21 19.04761905 | |
| 22 18.18181818 | |
| 23 17.39130435 | |
| 24 20.83333333 | |
| 25 20.00000000 | |
| 26 19.23076923 | |
| 27 18.51851852 | |
| 28 17.85714286 | |
| 29 20.68965517 | |
| 30 20.00000000 | |
| 31 19.35483871 | |
| 32 21.87500000 | |
| 33 24.24242424 | |
| 34 23.52941176 | |
| 35 22.85714286 | |
| 36 22.22222222 | |
| 37 24.32432432 | |
| 38 23.68421053 | |
| 39 25.64102564 | |
| 40 25.00000000 | |
| 41 24.39024390 | |
| 42 23.80952381 | |
| 43 25.58139535 | |
| 44 25.00000000 | |
| 45 26.66666667 | |
| 46 26.08695652 | |
| 47 25.53191489 | |
| 48 25.00000000 | |
| 49 24.48979592 | |
| 50 26.00000000 | |
| 51 27.45098039 | |
| 52 26.92307692 | |
| 53 26.41509434 | |
| 54 27.77777778 | |
| 55 29.09090909 | |
| 56 28.57142857 | |
| 57 28.07017544 | |
| 58 27.58620690 | |
| 59 27.11864407 | |
| 60 26.66666667 | |
| 61 26.22950820 | |
| 62 27.41935484 | |
| 63 26.98412698 | |
| 64 26.56250000 | |
| 65 26.15384615 | |
| 66 25.75757576 | |
| 67 25.37313433 | |
| 68 25.00000000 | |
| 69 26.08695652 | |
| 70 27.14285714 | |
| 71 26.76056338 | |
| 72 27.77777778 | |
| 73 27.39726027 | |
| 74 27.02702703 | |
| 75 26.66666667 | |
| 76 26.31578947 | |
| 77 25.97402597 | |
| 78 25.64102564 | |
| 79 25.31645570 | |
| 80 26.25000000 | |
| 81 25.92592593 | |
| 82 26.82926829 | |
| 83 26.50602410 | |
| 84 26.19047619 | |
| 85 25.88235294 | |
| 86 25.58139535 | |
| 87 25.28735632 | |
| 88 25.00000000 | |
| 89 25.84269663 | |
| 90 25.55555556 | |
| 91 26.37362637 | |
| 92 26.08695652 | |
| 93 25.80645161 | |
| 94 25.53191489 | |
| 95 26.31578947 | |
| 96 27.08333333 | |
| 97 26.80412371 | |
| 98 26.53061224 | |
| 99 27.27272727 | |
| 100 28.00000000 | |
| 101 28.71287129 | |
| 102 28.43137255 | |
| 103 28.15533981 | |
| 104 27.88461538 | |
| 105 27.61904762 | |
| 106 27.35849057 | |
| 107 28.03738318 | |
| 108 28.70370370 | |
| 109 28.44036697 | |
| 110 29.09090909 | |
| 111 28.82882883 | |
| 112 28.57142857 | |
| 113 28.31858407 | |
| 114 28.94736842 | |
| 115 28.69565217 | |
| 116 28.44827586 | |
| 117 28.20512821 | |
| 118 27.96610169 | |
| 119 27.73109244 | |
| 120 28.33333333 | |
| 121 28.92561983 | |
| 122 28.68852459 | |
| 123 29.26829268 | |
| 124 29.03225806 | |
| 125 29.60000000 | |
| 126 29.36507937 | |
| 127 29.13385827 | |
| 128 28.90625000 | |
| 129 28.68217054 | |
| 130 28.46153846 | |
| 131 28.24427481 | |
| 132 28.03030303 | |
| 133 27.81954887 | |
| 134 28.35820896 | |
| 135 28.88888889 | |
| 136 28.67647059 | |
| 137 29.19708029 | |
| 138 29.71014493 | |
| 139 30.21582734 | |
| 140 30.00000000 | |
| 141 29.78723404 | |
| 142 29.57746479 | |
| 143 29.37062937 | |
| 144 29.86111111 | |
| 145 29.65517241 | |
| 146 29.45205479 | |
| 147 29.25170068 | |
| 148 29.05405405 | |
| 149 28.85906040 | |
| 150 28.66666667 | |
| 151 29.13907285 | |
| 152 28.94736842 | |
| 153 28.75816993 | |
| 154 28.57142857 | |
| 155 29.03225806 | |
| 156 29.48717949 | |
| 157 29.29936306 | |
| 158 29.74683544 | |
| 159 29.55974843 | |
| 160 29.37500000 | |
| 161 29.19254658 | |
| 162 29.01234568 | |
| 163 28.83435583 | |
| 164 28.65853659 | |
| 165 29.09090909 | |
| 166 29.51807229 | |
| 167 29.34131737 | |
| 168 29.16666667 | |
| 169 28.99408284 | |
| 170 28.82352941 | |
| 171 28.65497076 | |
| 172 28.48837209 | |
| 173 28.32369942 | |
| 174 28.73563218 | |
| 175 28.57142857 | |
| 176 28.40909091 | |
| 177 28.24858757 | |
| 178 28.08988764 | |
| 179 28.49162011 | |
| 180 28.33333333 | |
| 181 28.17679558 | |
| 182 28.02197802 | |
| 183 27.86885246 | |
| 184 27.71739130 | |
| 185 27.56756757 | |
| 186 27.95698925 | |
| 187 27.80748663 | |
| 188 28.19148936 | |
| 189 28.57142857 | |
| 190 28.42105263 | |
| 191 28.27225131 | |
| 192 28.64583333 | |
| 193 28.49740933 | |
| 194 28.86597938 | |
| 195 28.71794872 | |
| 196 28.57142857 | |
| 197 28.93401015 | |
| 198 29.29292929 | |
| 199 29.14572864 | |
| 200 29.00000000 | |
| 201 28.85572139 | |
| 202 29.20792079 | |
| 203 29.06403941 | |
| 204 28.92156863 | |
| 205 28.78048780 | |
| 206 28.64077670 | |
| 207 28.98550725 | |
| 208 28.84615385 | |
| 209 28.70813397 | |
| 210 28.57142857 | |
| 211 28.43601896 | |
| 212 28.30188679 | |
| 213 28.16901408 | |
| 214 28.03738318 | |
| 215 27.90697674 | |
| 216 28.24074074 | |
| 217 28.11059908 | |
| 218 28.44036697 | |
| 219 28.31050228 | |
| 220 28.18181818 | |
| 221 28.05429864 | |
| 222 27.92792793 | |
| 223 27.80269058 | |
| 224 28.12500000 | |
| 225 28.00000000 | |
| 226 27.87610619 | |
| 227 28.19383260 | |
| 228 28.07017544 | |
| 229 27.94759825 | |
| 230 27.82608696 | |
| 231 27.70562771 | |
| 232 28.01724138 | |
| 233 28.32618026 | |
| 234 28.63247863 | |
| 235 28.51063830 | |
| 236 28.81355932 | |
| 237 29.11392405 | |
| 238 28.99159664 | |
| 239 28.87029289 | |
| 240 28.75000000 | |
| 241 28.63070539 | |
| 242 28.51239669 | |
| 243 28.39506173 | |
| 244 28.27868852 | |
| 245 28.16326531 | |
| 246 28.04878049 | |
| 247 27.93522267 | |
| 248 27.82258065 | |
| 249 27.71084337 | |
| 250 27.60000000 | |
| 251 27.88844622 | |
| 252 27.77777778 | |
| 253 27.66798419 | |
| 254 27.55905512 | |
| 255 27.84313725 | |
| 256 27.73437500 | |
| 257 27.62645914 | |
| 258 27.51937984 | |
| 259 27.41312741 | |
| 260 27.30769231 | |
| 261 27.58620690 | |
| 262 27.48091603 | |
| 263 27.37642586 | |
| 264 27.65151515 | |
| 265 27.92452830 | |
| 266 27.81954887 | |
| 267 28.08988764 | |
| 268 27.98507463 | |
| 269 27.88104089 | |
| 270 27.77777778 | |
| 271 27.67527675 | |
| 272 27.57352941 | |
| 273 27.47252747 | |
| 274 27.73722628 | |
| 275 27.63636364 | |
| 276 27.53623188 | |
| 277 27.43682310 | |
| 278 27.33812950 | |
| 279 27.24014337 | |
| 280 27.50000000 | |
| 281 27.75800712 | |
| 282 27.65957447 | |
| 283 27.56183746 | |
| 284 27.81690141 | |
| 285 27.71929825 | |
| 286 27.97202797 | |
| 287 28.22299652 | |
| 288 28.12500000 | |
| 289 28.02768166 | |
| 290 27.93103448 | |
| 291 28.17869416 | |
| 292 28.08219178 | |
| 293 28.32764505 | |
| 294 28.23129252 | |
| 295 28.47457627 | |
| 296 28.71621622 | |
| 297 28.95622896 | |
| 298 28.85906040 | |
| 299 28.76254181 | |
| 300 28.66666667 | |
| 301 28.57142857 | |
| 302 28.47682119 | |
| 303 28.38283828 | |
| 304 28.28947368 | |
| 305 28.19672131 | |
| 306 28.10457516 | |
| 307 28.01302932 | |
| 308 28.24675325 | |
| 309 28.15533981 | |
| 310 28.06451613 | |
| 311 27.97427653 | |
| 312 28.20512821 | |
| 313 28.11501597 | |
| 314 28.34394904 | |
| 315 28.25396825 | |
| 316 28.16455696 | |
| 317 28.07570978 | |
| 318 27.98742138 | |
| 319 27.89968652 | |
| 320 27.81250000 | |
| 321 27.72585670 | |
| 322 27.95031056 | |
| 323 27.86377709 | |
| 324 27.77777778 | |
| 325 27.69230769 | |
| 326 27.60736196 | |
| 327 27.82874618 | |
| 328 28.04878049 | |
| 329 27.96352584 | |
| 330 28.18181818 | |
| 331 28.09667674 | |
| 332 28.31325301 | |
| 333 28.22822823 | |
| 334 28.14371257 | |
| 335 28.35820896 | |
| 336 28.57142857 | |
| 337 28.48664688 | |
| 338 28.40236686 | |
| 339 28.61356932 | |
| 340 28.52941176 | |
| 341 28.73900293 | |
| 342 28.65497076 | |
| 343 28.57142857 | |
| 344 28.77906977 | |
| 345 28.98550725 | |
| 346 29.19075145 | |
| 347 29.39481268 | |
| 348 29.31034483 | |
| 349 29.22636103 | |
| 350 29.14285714 | |
| 351 29.05982906 | |
| 352 28.97727273 | |
| 353 28.89518414 | |
| 354 28.81355932 | |
| 355 29.01408451 | |
| 356 28.93258427 | |
| 357 29.13165266 | |
| 358 29.32960894 | |
| 359 29.24791086 | |
| 360 29.16666667 | |
| 361 29.36288089 | |
| 362 29.28176796 | |
| 363 29.47658402 | |
| 364 29.39560440 | |
| 365 29.58904110 | |
| 366 29.50819672 | |
| 367 29.70027248 | |
| 368 29.61956522 | |
| 369 29.53929539 | |
| 370 29.45945946 | |
| 371 29.38005391 | |
| 372 29.56989247 | |
| 373 29.49061662 | |
| 374 29.67914439 | |
| 375 29.60000000 | |
| 376 29.52127660 | |
| 377 29.44297082 | |
| 378 29.36507937 | |
| 379 29.28759894 | |
| 380 29.47368421 | |
| 381 29.39632546 | |
| 382 29.58115183 | |
| 383 29.50391645 | |
| 384 29.42708333 | |
| 385 29.35064935 | |
| 386 29.27461140 | |
| 387 29.19896641 | |
| 388 29.38144330 | |
| 389 29.30591260 | |
| 390 29.23076923 | |
| 391 29.15601023 | |
| 392 29.08163265 | |
| 393 29.26208651 | |
| 394 29.44162437 | |
| 395 29.36708861 | |
| 396 29.29292929 | |
| 397 29.47103275 | |
| 398 29.39698492 | |
| 399 29.32330827 | |
| 400 29.25000000 | |
| 401 29.17705736 | |
| 402 29.35323383 | |
| 403 29.52853598 | |
| 404 29.45544554 | |
| 405 29.38271605 | |
| 406 29.55665025 | |
| 407 29.48402948 | |
| 408 29.41176471 | |
| 409 29.33985330 | |
| 410 29.26829268 | |
| 411 29.19708029 | |
| 412 29.36893204 | |
| 413 29.29782082 | |
| 414 29.46859903 | |
| 415 29.63855422 | |
| 416 29.56730769 | |
| 417 29.49640288 | |
| 418 29.42583732 | |
| 419 29.35560859 | |
| 420 29.52380952 | |
| 421 29.69121140 | |
| 422 29.62085308 | |
| 423 29.55082742 | |
| 424 29.48113208 | |
| 425 29.41176471 | |
| 426 29.34272300 | |
| 427 29.27400468 | |
| 428 29.20560748 | |
| 429 29.37062937 | |
| 430 29.30232558 | |
| 431 29.23433875 | |
| 432 29.39814815 | |
| 433 29.56120092 | |
| 434 29.72350230 | |
| 435 29.65517241 | |
| 436 29.58715596 | |
| 437 29.51945080 | |
| 438 29.45205479 | |
| 439 29.38496583 | |
| 440 29.31818182 | |
| 441 29.25170068 | |
| 442 29.18552036 | |
| 443 29.11963883 | |
| 444 29.27927928 | |
| 445 29.21348315 | |
| 446 29.14798206 | |
| 447 29.08277405 | |
| 448 29.01785714 | |
| 449 29.17594655 | |
| 450 29.33333333 | |
| 451 29.26829268 | |
| 452 29.20353982 | |
| 453 29.13907285 | |
| 454 29.29515419 | |
| 455 29.23076923 | |
| 456 29.16666667 | |
| 457 29.10284464 | |
| 458 29.03930131 | |
| 459 28.97603486 | |
| 460 28.91304348 | |
| 461 28.85032538 | |
| 462 28.78787879 | |
| 463 28.94168467 | |
| 464 29.09482759 | |
| 465 29.24731183 | |
| 466 29.18454936 | |
| 467 29.12205567 | |
| 468 29.27350427 | |
| 469 29.21108742 | |
| 470 29.14893617 | |
| 471 29.29936306 | |
| 472 29.23728814 | |
| 473 29.17547569 | |
| 474 29.11392405 | |
| 475 29.26315789 | |
| 476 29.20168067 | |
| 477 29.35010482 | |
| 478 29.28870293 | |
| 479 29.22755741 | |
| 480 29.16666667 | |
| 481 29.10602911 | |
| 482 29.25311203 | |
| 483 29.19254658 | |
| 484 29.33884298 | |
| 485 29.48453608 | |
| 486 29.42386831 | |
| 487 29.56878850 | |
| 488 29.50819672 | |
| 489 29.65235174 | |
| 490 29.79591837 | |
| 491 29.93890020 | |
| 492 30.08130081 | |
| 493 30.02028398 | |
| 494 29.95951417 | |
| 495 30.10101010 | |
| 496 30.04032258 | |
| 497 29.97987928 | |
| 498 29.91967871 | |
| 499 30.06012024 | |
| 500 30.00000000 | |
| 501 30.13972056 | |
| 502 30.07968127 | |
| 503 30.01988072 | |
| 504 30.15873016 | |
| 505 30.09900990 | |
| 506 30.03952569 | |
| 507 30.17751479 | |
| 508 30.31496063 | |
| 509 30.45186640 | |
| 510 30.58823529 | |
| 511 30.52837573 | |
| 512 30.46875000 | |
| 513 30.40935673 | |
| 514 30.35019455 | |
| 515 30.48543689 | |
| 516 30.42635659 | |
| 517 30.56092843 | |
| 518 30.50193050 | |
| 519 30.44315992 | |
| 520 30.38461538 | |
| 521 30.32629559 | |
| 522 30.26819923 | |
| 523 30.21032505 | |
| 524 30.15267176 | |
| 525 30.09523810 | |
| 526 30.03802281 | |
| 527 30.17077799 | |
| 528 30.11363636 | |
| 529 30.05671078 | |
| 530 30.00000000 | |
| 531 29.94350282 | |
| 532 29.88721805 | |
| 533 30.01876173 | |
| 534 29.96254682 | |
| 535 29.90654206 | |
| 536 29.85074627 | |
| 537 29.98137803 | |
| 538 29.92565056 | |
| 539 29.87012987 | |
| 540 29.81481481 | |
| 541 29.75970425 | |
| 542 29.70479705 | |
| 543 29.65009208 | |
| 544 29.59558824 | |
| 545 29.54128440 | |
| 546 29.67032967 | |
| 547 29.79890311 | |
| 548 29.92700730 | |
| 549 29.87249545 | |
| 550 29.81818182 | |
| 551 29.76406534 | |
| 552 29.89130435 | |
| 553 30.01808318 | |
| 554 29.96389892 | |
| 555 29.90990991 | |
| 556 29.85611511 | |
| 557 29.80251346 | |
| 558 29.74910394 | |
| 559 29.69588551 | |
| 560 29.82142857 | |
| 561 29.94652406 | |
| 562 29.89323843 | |
| 563 29.84014210 | |
| 564 29.96453901 | |
| 565 30.08849558 | |
| 566 30.21201413 | |
| 567 30.15873016 | |
| 568 30.28169014 | |
| 569 30.40421793 | |
| 570 30.35087719 | |
| 571 30.29772329 | |
| 572 30.24475524 | |
| 573 30.19197208 | |
| 574 30.31358885 | |
| 575 30.43478261 | |
| 576 30.38194444 | |
| 577 30.32928943 | |
| 578 30.44982699 | |
| 579 30.39723661 | |
| 580 30.51724138 | |
| 581 30.46471601 | |
| 582 30.58419244 | |
| 583 30.53173242 | |
| 584 30.65068493 | |
| 585 30.59829060 | |
| 586 30.71672355 | |
| 587 30.66439523 | |
| 588 30.61224490 | |
| 589 30.56027165 | |
| 590 30.67796610 | |
| 591 30.62605753 | |
| 592 30.57432432 | |
| 593 30.52276560 | |
| 594 30.47138047 | |
| 595 30.42016807 | |
| 596 30.36912752 | |
| 597 30.31825796 | |
| 598 30.26755853 | |
| 599 30.21702838 | |
| 600 30.16666667 | |
| 601 30.28286190 | |
| 602 30.23255814 | |
| 603 30.34825871 | |
| 604 30.29801325 | |
| 605 30.24793388 | |
| 606 30.36303630 | |
| 607 30.31301483 | |
| 608 30.26315789 | |
| 609 30.21346470 | |
| 610 30.16393443 | |
| 611 30.11456628 | |
| 612 30.22875817 | |
| 613 30.17944535 | |
| 614 30.13029316 | |
| 615 30.08130081 | |
| 616 30.19480519 | |
| 617 30.30794165 | |
| 618 30.25889968 | |
| 619 30.37156704 | |
| 620 30.32258065 | |
| 621 30.27375201 | |
| 622 30.38585209 | |
| 623 30.33707865 | |
| 624 30.28846154 | |
| 625 30.24000000 | |
| 626 30.19169329 | |
| 627 30.30303030 | |
| 628 30.25477707 | |
| 629 30.20667727 | |
| 630 30.15873016 | |
| 631 30.26941363 | |
| 632 30.22151899 | |
| 633 30.17377567 | |
| 634 30.28391167 | |
| 635 30.23622047 | |
| 636 30.18867925 | |
| 637 30.14128728 | |
| 638 30.09404389 | |
| 639 30.04694836 | |
| 640 30.00000000 | |
| 641 29.95319813 | |
| 642 30.06230530 | |
| 643 30.01555210 | |
| 644 29.96894410 | |
| 645 30.07751938 | |
| 646 30.03095975 | |
| 647 29.98454405 | |
| 648 30.09259259 | |
| 649 30.04622496 | |
| 650 30.15384615 | |
| 651 30.26113671 | |
| 652 30.36809816 | |
| 653 30.47473201 | |
| 654 30.42813456 | |
| 655 30.38167939 | |
| 656 30.48780488 | |
| 657 30.59360731 | |
| 658 30.54711246 | |
| 659 30.65250379 | |
| 660 30.60606061 | |
| 661 30.55975794 | |
| 662 30.51359517 | |
| 663 30.46757164 | |
| 664 30.42168675 | |
| 665 30.37593985 | |
| 666 30.48048048 | |
| 667 30.43478261 | |
| 668 30.38922156 | |
| 669 30.34379671 | |
| 670 30.44776119 | |
| 671 30.40238450 | |
| 672 30.35714286 | |
| 673 30.31203566 | |
| 674 30.26706231 | |
| 675 30.37037037 | |
| 676 30.47337278 | |
| 677 30.42836041 | |
| 678 30.38348083 | |
| 679 30.33873343 | |
| 680 30.29411765 | |
| 681 30.24963289 | |
| 682 30.20527859 | |
| 683 30.16105417 | |
| 684 30.26315789 | |
| 685 30.21897810 | |
| 686 30.17492711 | |
| 687 30.13100437 | |
| 688 30.08720930 | |
| 689 30.18867925 | |
| 690 30.14492754 | |
| 691 30.24602026 | |
| 692 30.34682081 | |
| 693 30.44733045 | |
| 694 30.40345821 | |
| 695 30.35971223 | |
| 696 30.45977011 | |
| 697 30.41606887 | |
| 698 30.37249284 | |
| 699 30.32904149 | |
| 700 30.28571429 | |
| 701 30.24251070 | |
| 702 30.19943020 | |
| 703 30.15647226 | |
| 704 30.11363636 | |
| 705 30.07092199 | |
| 706 30.02832861 | |
| 707 30.12729844 | |
| 708 30.08474576 | |
| 709 30.04231312 | |
| 710 30.00000000 | |
| 711 29.95780591 | |
| 712 29.91573034 | |
| 713 29.87377279 | |
| 714 29.97198880 | |
| 715 30.06993007 | |
| 716 30.02793296 | |
| 717 29.98605300 | |
| 718 29.94428969 | |
| 719 29.90264256 | |
| 720 29.86111111 | |
| 721 29.81969487 | |
| 722 29.77839335 | |
| 723 29.73720609 | |
| 724 29.83425414 | |
| 725 29.79310345 | |
| 726 29.88980716 | |
| 727 29.84869326 | |
| 728 29.80769231 | |
| 729 29.90397805 | |
| 730 29.86301370 | |
| 731 29.82216142 | |
| 732 29.91803279 | |
| 733 29.87721692 | |
| 734 29.83651226 | |
| 735 29.79591837 | |
| 736 29.75543478 | |
| 737 29.85074627 | |
| 738 29.81029810 | |
| 739 29.90527740 | |
| 740 29.86486486 | |
| 741 29.95951417 | |
| 742 29.91913747 | |
| 743 29.87886945 | |
| 744 29.97311828 | |
| 745 29.93288591 | |
| 746 29.89276139 | |
| 747 29.98661312 | |
| 748 29.94652406 | |
| 749 30.04005340 | |
| 750 30.13333333 | |
| Final result: 30.1333 +/- 1.6766 | |
| Random chance: 19.8992 +/- 1.4588 | |