| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 0.00000000 | |
| 2 0.00000000 | |
| 3 0.00000000 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 28.57142857 | |
| 8 37.50000000 | |
| 9 44.44444444 | |
| 10 40.00000000 | |
| 11 36.36363636 | |
| 12 41.66666667 | |
| 13 46.15384615 | |
| 14 42.85714286 | |
| 15 46.66666667 | |
| 16 50.00000000 | |
| 17 52.94117647 | |
| 18 50.00000000 | |
| 19 52.63157895 | |
| 20 50.00000000 | |
| 21 47.61904762 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 45.83333333 | |
| 25 48.00000000 | |
| 26 46.15384615 | |
| 27 48.14814815 | |
| 28 46.42857143 | |
| 29 48.27586207 | |
| 30 50.00000000 | |
| 31 51.61290323 | |
| 32 53.12500000 | |
| 33 51.51515152 | |
| 34 52.94117647 | |
| 35 54.28571429 | |
| 36 52.77777778 | |
| 37 54.05405405 | |
| 38 52.63157895 | |
| 39 51.28205128 | |
| 40 50.00000000 | |
| 41 48.78048780 | |
| 42 47.61904762 | |
| 43 48.83720930 | |
| 44 47.72727273 | |
| 45 46.66666667 | |
| 46 45.65217391 | |
| 47 44.68085106 | |
| 48 45.83333333 | |
| 49 44.89795918 | |
| 50 44.00000000 | |
| 51 43.13725490 | |
| 52 42.30769231 | |
| 53 41.50943396 | |
| 54 40.74074074 | |
| 55 41.81818182 | |
| 56 41.07142857 | |
| 57 40.35087719 | |
| 58 41.37931034 | |
| 59 42.37288136 | |
| 60 41.66666667 | |
| 61 40.98360656 | |
| 62 40.32258065 | |
| 63 39.68253968 | |
| 64 39.06250000 | |
| 65 38.46153846 | |
| 66 39.39393939 | |
| 67 38.80597015 | |
| 68 38.23529412 | |
| 69 37.68115942 | |
| 70 38.57142857 | |
| 71 39.43661972 | |
| 72 38.88888889 | |
| 73 38.35616438 | |
| 74 39.18918919 | |
| 75 40.00000000 | |
| 76 40.78947368 | |
| 77 41.55844156 | |
| 78 42.30769231 | |
| 79 41.77215190 | |
| 80 42.50000000 | |
| 81 43.20987654 | |
| 82 42.68292683 | |
| 83 43.37349398 | |
| 84 44.04761905 | |
| 85 43.52941176 | |
| 86 43.02325581 | |
| 87 43.67816092 | |
| 88 44.31818182 | |
| 89 43.82022472 | |
| 90 43.33333333 | |
| 91 42.85714286 | |
| 92 42.39130435 | |
| 93 41.93548387 | |
| 94 42.55319149 | |
| 95 42.10526316 | |
| 96 42.70833333 | |
| 97 42.26804124 | |
| 98 41.83673469 | |
| 99 41.41414141 | |
| 100 41.00000000 | |
| 101 41.58415842 | |
| 102 41.17647059 | |
| 103 40.77669903 | |
| 104 41.34615385 | |
| 105 41.90476190 | |
| 106 42.45283019 | |
| 107 42.05607477 | |
| 108 41.66666667 | |
| 109 41.28440367 | |
| 110 40.90909091 | |
| 111 40.54054054 | |
| 112 41.07142857 | |
| 113 40.70796460 | |
| 114 41.22807018 | |
| 115 41.73913043 | |
| 116 41.37931034 | |
| 117 41.02564103 | |
| 118 40.67796610 | |
| 119 41.17647059 | |
| 120 41.66666667 | |
| 121 42.14876033 | |
| 122 41.80327869 | |
| 123 41.46341463 | |
| 124 41.12903226 | |
| 125 41.60000000 | |
| 126 41.26984127 | |
| 127 40.94488189 | |
| 128 40.62500000 | |
| 129 40.31007752 | |
| 130 40.00000000 | |
| 131 40.45801527 | |
| 132 40.15151515 | |
| 133 39.84962406 | |
| 134 39.55223881 | |
| 135 39.25925926 | |
| 136 38.97058824 | |
| 137 39.41605839 | |
| 138 39.13043478 | |
| 139 39.56834532 | |
| 140 40.00000000 | |
| 141 39.71631206 | |
| 142 40.14084507 | |
| 143 40.55944056 | |
| 144 40.27777778 | |
| 145 40.00000000 | |
| 146 40.41095890 | |
| 147 40.13605442 | |
| 148 40.54054054 | |
| 149 40.93959732 | |
| 150 40.66666667 | |
| 151 40.39735099 | |
| 152 40.78947368 | |
| 153 40.52287582 | |
| 154 40.25974026 | |
| 155 40.00000000 | |
| 156 39.74358974 | |
| 157 40.12738854 | |
| 158 40.50632911 | |
| 159 40.25157233 | |
| 160 40.62500000 | |
| 161 40.37267081 | |
| 162 40.12345679 | |
| 163 39.87730061 | |
| 164 39.63414634 | |
| 165 40.00000000 | |
| 166 39.75903614 | |
| 167 40.11976048 | |
| 168 39.88095238 | |
| 169 40.23668639 | |
| 170 40.00000000 | |
| 171 39.76608187 | |
| 172 39.53488372 | |
| 173 39.30635838 | |
| 174 39.65517241 | |
| 175 39.42857143 | |
| 176 39.20454545 | |
| 177 38.98305085 | |
| 178 39.32584270 | |
| 179 39.66480447 | |
| 180 39.44444444 | |
| 181 39.22651934 | |
| 182 39.01098901 | |
| 183 39.34426230 | |
| 184 39.13043478 | |
| 185 39.45945946 | |
| 186 39.78494624 | |
| 187 39.57219251 | |
| 188 39.36170213 | |
| 189 39.68253968 | |
| 190 39.47368421 | |
| 191 39.79057592 | |
| 192 39.58333333 | |
| 193 39.37823834 | |
| 194 39.17525773 | |
| 195 39.48717949 | |
| 196 39.28571429 | |
| 197 39.59390863 | |
| 198 39.39393939 | |
| 199 39.19597990 | |
| 200 39.00000000 | |
| 201 38.80597015 | |
| 202 38.61386139 | |
| 203 38.42364532 | |
| 204 38.23529412 | |
| 205 38.53658537 | |
| 206 38.34951456 | |
| 207 38.64734300 | |
| 208 38.46153846 | |
| 209 38.27751196 | |
| 210 38.09523810 | |
| 211 38.38862559 | |
| 212 38.20754717 | |
| 213 38.02816901 | |
| 214 37.85046729 | |
| 215 38.13953488 | |
| 216 38.42592593 | |
| 217 38.24884793 | |
| 218 38.53211009 | |
| 219 38.81278539 | |
| 220 39.09090909 | |
| 221 39.36651584 | |
| 222 39.63963964 | |
| 223 39.91031390 | |
| 224 40.17857143 | |
| 225 40.44444444 | |
| 226 40.70796460 | |
| 227 40.52863436 | |
| 228 40.78947368 | |
| 229 40.61135371 | |
| 230 40.43478261 | |
| 231 40.25974026 | |
| 232 40.08620690 | |
| 233 39.91416309 | |
| 234 40.17094017 | |
| 235 40.00000000 | |
| 236 39.83050847 | |
| 237 39.66244726 | |
| 238 39.49579832 | |
| 239 39.33054393 | |
| 240 39.16666667 | |
| 241 39.41908714 | |
| 242 39.66942149 | |
| 243 39.91769547 | |
| 244 39.75409836 | |
| 245 40.00000000 | |
| 246 39.83739837 | |
| 247 40.08097166 | |
| 248 40.32258065 | |
| 249 40.16064257 | |
| 250 40.00000000 | |
| 251 39.84063745 | |
| 252 39.68253968 | |
| 253 39.92094862 | |
| 254 39.76377953 | |
| 255 39.60784314 | |
| 256 39.84375000 | |
| 257 40.07782101 | |
| 258 39.92248062 | |
| 259 39.76833977 | |
| 260 39.61538462 | |
| 261 39.46360153 | |
| 262 39.31297710 | |
| 263 39.16349810 | |
| 264 39.39393939 | |
| 265 39.24528302 | |
| 266 39.47368421 | |
| 267 39.32584270 | |
| 268 39.17910448 | |
| 269 39.40520446 | |
| 270 39.62962963 | |
| 271 39.48339483 | |
| 272 39.33823529 | |
| 273 39.19413919 | |
| 274 39.05109489 | |
| 275 38.90909091 | |
| 276 39.13043478 | |
| 277 39.35018051 | |
| 278 39.56834532 | |
| 279 39.42652330 | |
| 280 39.64285714 | |
| 281 39.50177936 | |
| 282 39.36170213 | |
| 283 39.22261484 | |
| 284 39.08450704 | |
| 285 38.94736842 | |
| 286 38.81118881 | |
| 287 39.02439024 | |
| 288 38.88888889 | |
| 289 38.75432526 | |
| 290 38.96551724 | |
| 291 39.17525773 | |
| 292 39.38356164 | |
| 293 39.59044369 | |
| 294 39.45578231 | |
| 295 39.32203390 | |
| 296 39.18918919 | |
| 297 39.05723906 | |
| 298 39.26174497 | |
| 299 39.46488294 | |
| 300 39.66666667 | |
| 301 39.53488372 | |
| 302 39.73509934 | |
| 303 39.60396040 | |
| 304 39.47368421 | |
| 305 39.67213115 | |
| 306 39.54248366 | |
| 307 39.73941368 | |
| 308 39.61038961 | |
| 309 39.48220065 | |
| 310 39.67741935 | |
| 311 39.87138264 | |
| 312 40.06410256 | |
| 313 40.25559105 | |
| 314 40.44585987 | |
| 315 40.63492063 | |
| 316 40.50632911 | |
| 317 40.69400631 | |
| 318 40.56603774 | |
| 319 40.75235110 | |
| 320 40.62500000 | |
| 321 40.49844237 | |
| 322 40.37267081 | |
| 323 40.24767802 | |
| 324 40.12345679 | |
| 325 40.30769231 | |
| 326 40.18404908 | |
| 327 40.36697248 | |
| 328 40.54878049 | |
| 329 40.72948328 | |
| 330 40.60606061 | |
| 331 40.48338369 | |
| 332 40.36144578 | |
| 333 40.24024024 | |
| 334 40.11976048 | |
| 335 40.29850746 | |
| 336 40.47619048 | |
| 337 40.35608309 | |
| 338 40.23668639 | |
| 339 40.11799410 | |
| 340 40.29411765 | |
| 341 40.46920821 | |
| 342 40.64327485 | |
| 343 40.52478134 | |
| 344 40.40697674 | |
| 345 40.28985507 | |
| 346 40.46242775 | |
| 347 40.63400576 | |
| 348 40.51724138 | |
| 349 40.68767908 | |
| 350 40.85714286 | |
| 351 40.74074074 | |
| 352 40.62500000 | |
| 353 40.79320113 | |
| 354 40.67796610 | |
| 355 40.56338028 | |
| 356 40.73033708 | |
| 357 40.61624650 | |
| 358 40.50279330 | |
| 359 40.38997214 | |
| 360 40.55555556 | |
| 361 40.44321330 | |
| 362 40.60773481 | |
| 363 40.77134986 | |
| 364 40.65934066 | |
| 365 40.54794521 | |
| 366 40.71038251 | |
| 367 40.59945504 | |
| 368 40.48913043 | |
| 369 40.37940379 | |
| 370 40.54054054 | |
| 371 40.43126685 | |
| 372 40.32258065 | |
| 373 40.21447721 | |
| 374 40.37433155 | |
| 375 40.53333333 | |
| 376 40.42553191 | |
| 377 40.31830239 | |
| 378 40.47619048 | |
| 379 40.36939314 | |
| 380 40.26315789 | |
| 381 40.41994751 | |
| 382 40.57591623 | |
| 383 40.73107050 | |
| 384 40.62500000 | |
| 385 40.77922078 | |
| 386 40.67357513 | |
| 387 40.56847545 | |
| 388 40.46391753 | |
| 389 40.61696658 | |
| 390 40.76923077 | |
| 391 40.92071611 | |
| 392 41.07142857 | |
| 393 40.96692112 | |
| 394 40.86294416 | |
| 395 41.01265823 | |
| 396 41.16161616 | |
| 397 41.05793451 | |
| 398 40.95477387 | |
| 399 41.10275689 | |
| 400 41.00000000 | |
| 401 41.14713217 | |
| 402 41.04477612 | |
| 403 40.94292804 | |
| 404 40.84158416 | |
| 405 40.74074074 | |
| 406 40.64039409 | |
| 407 40.54054054 | |
| 408 40.44117647 | |
| 409 40.58679707 | |
| 410 40.48780488 | |
| 411 40.38929440 | |
| 412 40.29126214 | |
| 413 40.19370460 | |
| 414 40.09661836 | |
| 415 40.24096386 | |
| 416 40.14423077 | |
| 417 40.04796163 | |
| 418 39.95215311 | |
| 419 40.09546539 | |
| 420 40.23809524 | |
| 421 40.14251781 | |
| 422 40.04739336 | |
| 423 39.95271868 | |
| 424 40.09433962 | |
| 425 40.23529412 | |
| 426 40.37558685 | |
| 427 40.28103044 | |
| 428 40.18691589 | |
| 429 40.32634033 | |
| 430 40.23255814 | |
| 431 40.37122970 | |
| 432 40.50925926 | |
| 433 40.41570439 | |
| 434 40.32258065 | |
| 435 40.22988506 | |
| 436 40.13761468 | |
| 437 40.04576659 | |
| 438 40.18264840 | |
| 439 40.31890661 | |
| 440 40.22727273 | |
| 441 40.13605442 | |
| 442 40.04524887 | |
| 443 39.95485327 | |
| 444 39.86486486 | |
| 445 39.77528090 | |
| 446 39.68609865 | |
| 447 39.82102908 | |
| 448 39.73214286 | |
| 449 39.64365256 | |
| 450 39.55555556 | |
| 451 39.46784922 | |
| 452 39.60176991 | |
| 453 39.73509934 | |
| 454 39.64757709 | |
| 455 39.56043956 | |
| 456 39.69298246 | |
| 457 39.82494530 | |
| 458 39.73799127 | |
| 459 39.65141612 | |
| 460 39.78260870 | |
| 461 39.69631236 | |
| 462 39.61038961 | |
| 463 39.52483801 | |
| 464 39.65517241 | |
| 465 39.56989247 | |
| 466 39.48497854 | |
| 467 39.61456103 | |
| 468 39.52991453 | |
| 469 39.65884861 | |
| 470 39.57446809 | |
| 471 39.49044586 | |
| 472 39.40677966 | |
| 473 39.32346723 | |
| 474 39.24050633 | |
| 475 39.15789474 | |
| 476 39.28571429 | |
| 477 39.20335430 | |
| 478 39.33054393 | |
| 479 39.45720251 | |
| 480 39.58333333 | |
| 481 39.50103950 | |
| 482 39.41908714 | |
| 483 39.33747412 | |
| 484 39.25619835 | |
| 485 39.17525773 | |
| 486 39.09465021 | |
| 487 39.21971253 | |
| 488 39.13934426 | |
| 489 39.05930470 | |
| 490 39.18367347 | |
| 491 39.10386965 | |
| 492 39.02439024 | |
| 493 38.94523327 | |
| 494 39.06882591 | |
| 495 39.19191919 | |
| 496 39.31451613 | |
| 497 39.23541247 | |
| 498 39.15662651 | |
| 499 39.07815631 | |
| 500 39.00000000 | |
| 501 38.92215569 | |
| 502 38.84462151 | |
| 503 38.76739563 | |
| 504 38.88888889 | |
| 505 38.81188119 | |
| 506 38.93280632 | |
| 507 38.85601578 | |
| 508 38.97637795 | |
| 509 38.89980354 | |
| 510 39.01960784 | |
| 511 38.94324853 | |
| 512 38.86718750 | |
| 513 38.79142300 | |
| 514 38.71595331 | |
| 515 38.64077670 | |
| 516 38.75968992 | |
| 517 38.87814313 | |
| 518 38.99613900 | |
| 519 38.92100193 | |
| 520 39.03846154 | |
| 521 38.96353167 | |
| 522 38.88888889 | |
| 523 38.81453155 | |
| 524 38.93129771 | |
| 525 38.85714286 | |
| 526 38.78326996 | |
| 527 38.89943074 | |
| 528 39.01515152 | |
| 529 38.94139887 | |
| 530 39.05660377 | |
| 531 38.98305085 | |
| 532 38.90977444 | |
| 533 38.83677298 | |
| 534 38.76404494 | |
| 535 38.87850467 | |
| 536 38.80597015 | |
| 537 38.91992551 | |
| 538 39.03345725 | |
| 539 38.96103896 | |
| 540 38.88888889 | |
| 541 38.81700555 | |
| 542 38.92988930 | |
| 543 38.85819521 | |
| 544 38.78676471 | |
| 545 38.71559633 | |
| 546 38.64468864 | |
| 547 38.57404022 | |
| 548 38.68613139 | |
| 549 38.61566485 | |
| 550 38.54545455 | |
| 551 38.65698730 | |
| 552 38.58695652 | |
| 553 38.51717902 | |
| 554 38.44765343 | |
| 555 38.37837838 | |
| 556 38.48920863 | |
| 557 38.42010772 | |
| 558 38.53046595 | |
| 559 38.46153846 | |
| 560 38.39285714 | |
| 561 38.32442068 | |
| 562 38.43416370 | |
| 563 38.54351687 | |
| 564 38.47517730 | |
| 565 38.58407080 | |
| 566 38.69257951 | |
| 567 38.62433862 | |
| 568 38.55633803 | |
| 569 38.48857645 | |
| 570 38.59649123 | |
| 571 38.52889667 | |
| 572 38.63636364 | |
| 573 38.56893543 | |
| 574 38.67595819 | |
| 575 38.60869565 | |
| 576 38.71527778 | |
| 577 38.82149047 | |
| 578 38.75432526 | |
| 579 38.68739206 | |
| 580 38.62068966 | |
| 581 38.55421687 | |
| 582 38.48797251 | |
| 583 38.59348199 | |
| 584 38.69863014 | |
| 585 38.63247863 | |
| 586 38.73720137 | |
| 587 38.67120954 | |
| 588 38.60544218 | |
| 589 38.53989813 | |
| 590 38.47457627 | |
| 591 38.40947547 | |
| 592 38.34459459 | |
| 593 38.27993255 | |
| 594 38.38383838 | |
| 595 38.31932773 | |
| 596 38.42281879 | |
| 597 38.35845896 | |
| 598 38.29431438 | |
| 599 38.39732888 | |
| 600 38.33333333 | |
| 601 38.43594010 | |
| 602 38.37209302 | |
| 603 38.47429519 | |
| 604 38.41059603 | |
| 605 38.34710744 | |
| 606 38.28382838 | |
| 607 38.38550247 | |
| 608 38.32236842 | |
| 609 38.25944171 | |
| 610 38.19672131 | |
| 611 38.29787234 | |
| 612 38.23529412 | |
| 613 38.17292007 | |
| 614 38.11074919 | |
| 615 38.04878049 | |
| 616 38.14935065 | |
| 617 38.08752026 | |
| 618 38.02588997 | |
| 619 37.96445880 | |
| 620 37.90322581 | |
| 621 37.84219002 | |
| 622 37.78135048 | |
| 623 37.72070626 | |
| 624 37.66025641 | |
| 625 37.60000000 | |
| 626 37.69968051 | |
| 627 37.63955343 | |
| 628 37.57961783 | |
| 629 37.67885533 | |
| 630 37.77777778 | |
| 631 37.87638669 | |
| 632 37.97468354 | |
| 633 37.91469194 | |
| 634 37.85488959 | |
| 635 37.79527559 | |
| 636 37.73584906 | |
| 637 37.83359498 | |
| 638 37.77429467 | |
| 639 37.87167449 | |
| 640 37.96875000 | |
| 641 37.90951638 | |
| 642 38.00623053 | |
| 643 37.94712286 | |
| 644 38.04347826 | |
| 645 37.98449612 | |
| 646 37.92569659 | |
| 647 37.86707883 | |
| 648 37.80864198 | |
| 649 37.75038521 | |
| 650 37.84615385 | |
| 651 37.78801843 | |
| 652 37.73006135 | |
| 653 37.67228178 | |
| 654 37.76758410 | |
| 655 37.70992366 | |
| 656 37.65243902 | |
| 657 37.59512938 | |
| 658 37.68996960 | |
| 659 37.63277693 | |
| 660 37.57575758 | |
| 661 37.51891074 | |
| 662 37.46223565 | |
| 663 37.40573152 | |
| 664 37.50000000 | |
| 665 37.59398496 | |
| 666 37.53753754 | |
| 667 37.63118441 | |
| 668 37.72455090 | |
| 669 37.81763827 | |
| 670 37.76119403 | |
| 671 37.70491803 | |
| 672 37.64880952 | |
| 673 37.74145617 | |
| 674 37.68545994 | |
| 675 37.62962963 | |
| 676 37.57396450 | |
| 677 37.51846381 | |
| 678 37.46312684 | |
| 679 37.40795287 | |
| 680 37.50000000 | |
| 681 37.59177680 | |
| 682 37.53665689 | |
| 683 37.48169839 | |
| 684 37.42690058 | |
| 685 37.37226277 | |
| 686 37.31778426 | |
| 687 37.26346434 | |
| 688 37.20930233 | |
| 689 37.30043541 | |
| 690 37.24637681 | |
| 691 37.33719247 | |
| 692 37.28323699 | |
| 693 37.22943723 | |
| 694 37.17579251 | |
| 695 37.12230216 | |
| 696 37.21264368 | |
| 697 37.30272597 | |
| 698 37.24928367 | |
| 699 37.33905579 | |
| 700 37.28571429 | |
| 701 37.23252496 | |
| 702 37.32193732 | |
| 703 37.41109531 | |
| 704 37.35795455 | |
| 705 37.30496454 | |
| 706 37.25212465 | |
| 707 37.19943423 | |
| 708 37.14689266 | |
| 709 37.23554302 | |
| 710 37.32394366 | |
| 711 37.41209564 | |
| 712 37.35955056 | |
| 713 37.44740533 | |
| 714 37.53501401 | |
| 715 37.48251748 | |
| 716 37.56983240 | |
| 717 37.51743375 | |
| 718 37.46518106 | |
| 719 37.55215577 | |
| 720 37.50000000 | |
| 721 37.44798890 | |
| 722 37.53462604 | |
| 723 37.62102351 | |
| 724 37.56906077 | |
| 725 37.65517241 | |
| 726 37.60330579 | |
| 727 37.55158184 | |
| 728 37.50000000 | |
| 729 37.44855967 | |
| 730 37.39726027 | |
| 731 37.34610123 | |
| 732 37.43169399 | |
| 733 37.51705321 | |
| 734 37.60217984 | |
| 735 37.55102041 | |
| 736 37.50000000 | |
| 737 37.58480326 | |
| 738 37.53387534 | |
| 739 37.61840325 | |
| 740 37.56756757 | |
| 741 37.51686910 | |
| 742 37.60107817 | |
| 743 37.68506057 | |
| 744 37.76881720 | |
| 745 37.71812081 | |
| 746 37.66756032 | |
| 747 37.61713521 | |
| 748 37.56684492 | |
| 749 37.51668892 | |
| 750 37.60000000 | |
| Final result: 37.6000 +/- 1.7699 | |
| Random chance: 25.0000 +/- 1.5822 | |