| { | |
| "metadata": { | |
| "timestamp": "2025-03-12T10:52:59.228266", | |
| "vidore_benchmark_version": "4.0.3.dev37+g1548c2d.d20250311" | |
| }, | |
| "metrics": { | |
| "vidore/restaurant_esg_reports_beir": { | |
| "ndcg_at_1": 0.46154, | |
| "ndcg_at_3": 0.51553, | |
| "ndcg_at_5": 0.56137, | |
| "ndcg_at_10": 0.59354, | |
| "ndcg_at_20": 0.63362, | |
| "ndcg_at_50": 0.65072, | |
| "ndcg_at_100": 0.66201, | |
| "map_at_1": 0.36346, | |
| "map_at_3": 0.44902, | |
| "map_at_5": 0.49142, | |
| "map_at_10": 0.5106, | |
| "map_at_20": 0.52664, | |
| "map_at_50": 0.53321, | |
| "map_at_100": 0.53551, | |
| "recall_at_1": 0.36346, | |
| "recall_at_3": 0.50948, | |
| "recall_at_5": 0.6369, | |
| "recall_at_10": 0.71978, | |
| "recall_at_20": 0.83934, | |
| "recall_at_50": 0.90182, | |
| "recall_at_100": 0.94766, | |
| "precision_at_1": 0.46154, | |
| "precision_at_3": 0.27564, | |
| "precision_at_5": 0.22308, | |
| "precision_at_10": 0.13462, | |
| "precision_at_20": 0.08462, | |
| "precision_at_50": 0.03962, | |
| "precision_at_100": 0.02173, | |
| "mrr_at_1": 0.46153846153846156, | |
| "mrr_at_3": 0.564102564102564, | |
| "mrr_at_5": 0.5910256410256409, | |
| "mrr_at_10": 0.6018620268620267, | |
| "mrr_at_20": 0.609330199714815, | |
| "mrr_at_50": 0.6099933296882898, | |
| "mrr_at_100": 0.6099933296882898, | |
| "naucs_at_1_max": 0.20368019444404006, | |
| "naucs_at_1_std": -0.05896301709840663, | |
| "naucs_at_1_diff1": 0.5027857837399152, | |
| "naucs_at_3_max": 0.2207596691773764, | |
| "naucs_at_3_std": 0.006572688085190094, | |
| "naucs_at_3_diff1": 0.07535489280712841, | |
| "naucs_at_5_max": 0.3055052564988994, | |
| "naucs_at_5_std": 0.2695070152185379, | |
| "naucs_at_5_diff1": 0.028735419474887416, | |
| "naucs_at_10_max": 0.13022275163938632, | |
| "naucs_at_10_std": 0.16804625913226817, | |
| "naucs_at_10_diff1": -0.1044735223259232, | |
| "naucs_at_20_max": 0.030677866275297932, | |
| "naucs_at_20_std": 0.2067178669184376, | |
| "naucs_at_20_diff1": -0.15547715926288114, | |
| "naucs_at_50_max": 0.06721043932684811, | |
| "naucs_at_50_std": 0.2814855535601418, | |
| "naucs_at_50_diff1": -0.15644474250949775, | |
| "naucs_at_100_max": 0.04140512572483361, | |
| "naucs_at_100_std": 0.2754882090192102, | |
| "naucs_at_100_diff1": -0.20499364042570448 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.50625, | |
| "ndcg_at_3": 0.53801, | |
| "ndcg_at_5": 0.56367, | |
| "ndcg_at_10": 0.59144, | |
| "ndcg_at_20": 0.62281, | |
| "ndcg_at_50": 0.64988, | |
| "ndcg_at_100": 0.6614, | |
| "map_at_1": 0.2947, | |
| "map_at_3": 0.42389, | |
| "map_at_5": 0.46584, | |
| "map_at_10": 0.49641, | |
| "map_at_20": 0.51381, | |
| "map_at_50": 0.52294, | |
| "map_at_100": 0.52512, | |
| "recall_at_1": 0.2947, | |
| "recall_at_3": 0.51066, | |
| "recall_at_5": 0.6115, | |
| "recall_at_10": 0.69677, | |
| "recall_at_20": 0.78713, | |
| "recall_at_50": 0.87152, | |
| "recall_at_100": 0.92041, | |
| "precision_at_1": 0.50625, | |
| "precision_at_3": 0.34583, | |
| "precision_at_5": 0.26625, | |
| "precision_at_10": 0.16813, | |
| "precision_at_20": 0.10125, | |
| "precision_at_50": 0.04987, | |
| "precision_at_100": 0.02706, | |
| "mrr_at_1": 0.50625, | |
| "mrr_at_3": 0.6187499999999999, | |
| "mrr_at_5": 0.6390625, | |
| "mrr_at_10": 0.6450694444444445, | |
| "mrr_at_20": 0.6481967685092686, | |
| "mrr_at_50": 0.6493743580027906, | |
| "mrr_at_100": 0.6497234378933107, | |
| "naucs_at_1_max": 0.2982323803776623, | |
| "naucs_at_1_std": -0.053774525827026284, | |
| "naucs_at_1_diff1": 0.5200020115159291, | |
| "naucs_at_3_max": 0.18024374308063665, | |
| "naucs_at_3_std": 0.07332710212526489, | |
| "naucs_at_3_diff1": 0.019165502096226788, | |
| "naucs_at_5_max": 0.14274071990595502, | |
| "naucs_at_5_std": 0.09730846621519294, | |
| "naucs_at_5_diff1": -0.00690192345139435, | |
| "naucs_at_10_max": 0.1869838718265326, | |
| "naucs_at_10_std": 0.17477047588380465, | |
| "naucs_at_10_diff1": -0.07438077296572282, | |
| "naucs_at_20_max": 0.24640900247147313, | |
| "naucs_at_20_std": 0.2484156281222067, | |
| "naucs_at_20_diff1": -0.1333943313877057, | |
| "naucs_at_50_max": 0.22134620285423043, | |
| "naucs_at_50_std": 0.3203443552497451, | |
| "naucs_at_50_diff1": -0.13630064984709492, | |
| "naucs_at_100_max": 0.18852863001883446, | |
| "naucs_at_100_std": 0.32571156994244094, | |
| "naucs_at_100_diff1": -0.11435432105284754 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.67241, | |
| "ndcg_at_3": 0.63319, | |
| "ndcg_at_5": 0.58758, | |
| "ndcg_at_10": 0.57198, | |
| "ndcg_at_20": 0.58918, | |
| "ndcg_at_50": 0.65786, | |
| "ndcg_at_100": 0.69503, | |
| "map_at_1": 0.13615, | |
| "map_at_3": 0.21649, | |
| "map_at_5": 0.25675, | |
| "map_at_10": 0.3252, | |
| "map_at_20": 0.38016, | |
| "map_at_50": 0.43771, | |
| "map_at_100": 0.46413, | |
| "recall_at_1": 0.13615, | |
| "recall_at_3": 0.23646, | |
| "recall_at_5": 0.29799, | |
| "recall_at_10": 0.43273, | |
| "recall_at_20": 0.56259, | |
| "recall_at_50": 0.76924, | |
| "recall_at_100": 0.88035, | |
| "precision_at_1": 0.67241, | |
| "precision_at_3": 0.57471, | |
| "precision_at_5": 0.48966, | |
| "precision_at_10": 0.39828, | |
| "precision_at_20": 0.29914, | |
| "precision_at_50": 0.19172, | |
| "precision_at_100": 0.12379, | |
| "mrr_at_1": 0.6724137931034483, | |
| "mrr_at_3": 0.7557471264367817, | |
| "mrr_at_5": 0.764367816091954, | |
| "mrr_at_10": 0.7718596059113301, | |
| "mrr_at_20": 0.7718596059113301, | |
| "mrr_at_50": 0.7732378352919377, | |
| "mrr_at_100": 0.7732378352919377, | |
| "naucs_at_1_max": 0.032282867498218674, | |
| "naucs_at_1_std": -0.28271414000179634, | |
| "naucs_at_1_diff1": 0.14417521370616468, | |
| "naucs_at_3_max": 0.19868656246441452, | |
| "naucs_at_3_std": 0.12756096614672727, | |
| "naucs_at_3_diff1": -0.14423513218105685, | |
| "naucs_at_5_max": 0.16258116633927042, | |
| "naucs_at_5_std": 0.08888872261627004, | |
| "naucs_at_5_diff1": -0.09215214837175129, | |
| "naucs_at_10_max": 0.08087197701067339, | |
| "naucs_at_10_std": 0.0593535492140523, | |
| "naucs_at_10_diff1": 0.011413698146931404, | |
| "naucs_at_20_max": 0.09066512825980459, | |
| "naucs_at_20_std": 0.1515192296500721, | |
| "naucs_at_20_diff1": 0.05635159011980896, | |
| "naucs_at_50_max": 0.07419981716098847, | |
| "naucs_at_50_std": 0.18498184475514726, | |
| "naucs_at_50_diff1": 0.041495490587423654, | |
| "naucs_at_100_max": 0.03526540983660958, | |
| "naucs_at_100_std": 0.11166203694040092, | |
| "naucs_at_100_diff1": 0.12019253855015599 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.42105, | |
| "ndcg_at_3": 0.446, | |
| "ndcg_at_5": 0.4719, | |
| "ndcg_at_10": 0.51181, | |
| "ndcg_at_20": 0.5424, | |
| "ndcg_at_50": 0.56432, | |
| "ndcg_at_100": 0.5872, | |
| "map_at_1": 0.18615, | |
| "map_at_3": 0.3094, | |
| "map_at_5": 0.34722, | |
| "map_at_10": 0.38335, | |
| "map_at_20": 0.40112, | |
| "map_at_50": 0.40968, | |
| "map_at_100": 0.41626, | |
| "recall_at_1": 0.18615, | |
| "recall_at_3": 0.42928, | |
| "recall_at_5": 0.52991, | |
| "recall_at_10": 0.66591, | |
| "recall_at_20": 0.76433, | |
| "recall_at_50": 0.83751, | |
| "recall_at_100": 0.91575, | |
| "precision_at_1": 0.42105, | |
| "precision_at_3": 0.30409, | |
| "precision_at_5": 0.24561, | |
| "precision_at_10": 0.16667, | |
| "precision_at_20": 0.10789, | |
| "precision_at_50": 0.05333, | |
| "precision_at_100": 0.03263, | |
| "mrr_at_1": 0.42105263157894735, | |
| "mrr_at_3": 0.5350877192982455, | |
| "mrr_at_5": 0.5552631578947367, | |
| "mrr_at_10": 0.5674463937621831, | |
| "mrr_at_20": 0.5723974683727004, | |
| "mrr_at_50": 0.5730024290502564, | |
| "mrr_at_100": 0.5730024290502564, | |
| "naucs_at_1_max": -0.1063878813786229, | |
| "naucs_at_1_std": 0.07009527096294454, | |
| "naucs_at_1_diff1": 0.10532570058239055, | |
| "naucs_at_3_max": -0.0707818319042163, | |
| "naucs_at_3_std": 0.10106279988011081, | |
| "naucs_at_3_diff1": 0.084845655655151, | |
| "naucs_at_5_max": -0.11487901190848457, | |
| "naucs_at_5_std": 0.09251915109792752, | |
| "naucs_at_5_diff1": 0.012535607370615548, | |
| "naucs_at_10_max": -0.09127037923640684, | |
| "naucs_at_10_std": 0.13093656784846772, | |
| "naucs_at_10_diff1": 0.03006202135754748, | |
| "naucs_at_20_max": -0.05693043074070981, | |
| "naucs_at_20_std": 0.1422048179530491, | |
| "naucs_at_20_diff1": 0.021927827197454224, | |
| "naucs_at_50_max": -0.12190658822908639, | |
| "naucs_at_50_std": 0.16430064525199148, | |
| "naucs_at_50_diff1": -0.10417541492717723, | |
| "naucs_at_100_max": -0.17484165527500634, | |
| "naucs_at_100_std": 0.13672197776470205, | |
| "naucs_at_100_diff1": -0.15839101235075956 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0": { | |
| "ndcg_at_1": 0.72222, | |
| "ndcg_at_3": 0.66954, | |
| "ndcg_at_5": 0.64088, | |
| "ndcg_at_10": 0.6477, | |
| "ndcg_at_20": 0.68679, | |
| "ndcg_at_50": 0.73939, | |
| "ndcg_at_100": 0.75782, | |
| "map_at_1": 0.36677, | |
| "map_at_3": 0.4592, | |
| "map_at_5": 0.48529, | |
| "map_at_10": 0.53077, | |
| "map_at_20": 0.56672, | |
| "map_at_50": 0.58988, | |
| "map_at_100": 0.5958, | |
| "recall_at_1": 0.36677, | |
| "recall_at_3": 0.50423, | |
| "recall_at_5": 0.5832, | |
| "recall_at_10": 0.66079, | |
| "recall_at_20": 0.77947, | |
| "recall_at_50": 0.9373, | |
| "recall_at_100": 0.99068, | |
| "precision_at_1": 0.72222, | |
| "precision_at_3": 0.44444, | |
| "precision_at_5": 0.32222, | |
| "precision_at_10": 0.23333, | |
| "precision_at_20": 0.15833, | |
| "precision_at_50": 0.08333, | |
| "precision_at_100": 0.04667, | |
| "mrr_at_1": 0.7222222222222222, | |
| "mrr_at_3": 0.7592592592592593, | |
| "mrr_at_5": 0.7842592592592593, | |
| "mrr_at_10": 0.7842592592592593, | |
| "mrr_at_20": 0.7842592592592593, | |
| "mrr_at_50": 0.7856481481481482, | |
| "mrr_at_100": 0.7856481481481482, | |
| "naucs_at_1_max": -0.5764508261912696, | |
| "naucs_at_1_std": 0.2572205519822324, | |
| "naucs_at_1_diff1": 0.5945012986070088, | |
| "naucs_at_3_max": -0.3245546955664279, | |
| "naucs_at_3_std": 0.2106608705520064, | |
| "naucs_at_3_diff1": -0.18354778613695244, | |
| "naucs_at_5_max": -0.32979775271957523, | |
| "naucs_at_5_std": 0.004295616052457978, | |
| "naucs_at_5_diff1": -0.21668629588661706, | |
| "naucs_at_10_max": -0.37966732415422166, | |
| "naucs_at_10_std": -0.01491020124585064, | |
| "naucs_at_10_diff1": -0.17349036538288762, | |
| "naucs_at_20_max": -0.29338316737381204, | |
| "naucs_at_20_std": -0.03576033793090156, | |
| "naucs_at_20_diff1": -0.19533219218843523, | |
| "naucs_at_50_max": -0.29990699257368786, | |
| "naucs_at_50_std": -0.11316959031750934, | |
| "naucs_at_50_diff1": -0.27924927011864364, | |
| "naucs_at_100_max": -0.22547374701018, | |
| "naucs_at_100_std": -0.1279133290410171, | |
| "naucs_at_100_diff1": -0.32901479145609697 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.41667, | |
| "ndcg_at_3": 0.44206, | |
| "ndcg_at_5": 0.46171, | |
| "ndcg_at_10": 0.50911, | |
| "ndcg_at_20": 0.54735, | |
| "ndcg_at_50": 0.5785, | |
| "ndcg_at_100": 0.59513, | |
| "map_at_1": 0.18884, | |
| "map_at_3": 0.30884, | |
| "map_at_5": 0.34205, | |
| "map_at_10": 0.38214, | |
| "map_at_20": 0.40339, | |
| "map_at_50": 0.41645, | |
| "map_at_100": 0.42174, | |
| "recall_at_1": 0.18884, | |
| "recall_at_3": 0.41439, | |
| "recall_at_5": 0.50081, | |
| "recall_at_10": 0.65362, | |
| "recall_at_20": 0.77708, | |
| "recall_at_50": 0.87498, | |
| "recall_at_100": 0.92944, | |
| "precision_at_1": 0.41667, | |
| "precision_at_3": 0.30409, | |
| "precision_at_5": 0.2386, | |
| "precision_at_10": 0.16667, | |
| "precision_at_20": 0.10965, | |
| "precision_at_50": 0.05763, | |
| "precision_at_100": 0.03364, | |
| "mrr_at_1": 0.4166666666666667, | |
| "mrr_at_3": 0.5387426900584793, | |
| "mrr_at_5": 0.5527777777777776, | |
| "mrr_at_10": 0.5685202589807851, | |
| "mrr_at_20": 0.5744233342337056, | |
| "mrr_at_50": 0.574930578048572, | |
| "mrr_at_100": 0.574930578048572, | |
| "naucs_at_1_max": 0.01715145223542086, | |
| "naucs_at_1_std": 0.0899310510434977, | |
| "naucs_at_1_diff1": 0.1739307636238112, | |
| "naucs_at_3_max": -0.031778970284482205, | |
| "naucs_at_3_std": 0.15113736191861826, | |
| "naucs_at_3_diff1": -0.023575716122440124, | |
| "naucs_at_5_max": -0.11291764491834187, | |
| "naucs_at_5_std": 0.10256509751947128, | |
| "naucs_at_5_diff1": -0.07511697723029634, | |
| "naucs_at_10_max": -0.06154590672598315, | |
| "naucs_at_10_std": 0.08064531970193034, | |
| "naucs_at_10_diff1": -0.018188695451970553, | |
| "naucs_at_20_max": -0.014776270960712531, | |
| "naucs_at_20_std": 0.17295501151323417, | |
| "naucs_at_20_diff1": -0.06161735463033105, | |
| "naucs_at_50_max": -0.05048590733744633, | |
| "naucs_at_50_std": 0.1490648093736084, | |
| "naucs_at_50_diff1": -0.13782527489285434, | |
| "naucs_at_100_max": -0.06129053077443416, | |
| "naucs_at_100_std": 0.1378046540638781, | |
| "naucs_at_100_diff1": -0.16944450675960782 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.65278, | |
| "ndcg_at_3": 0.61596, | |
| "ndcg_at_5": 0.59512, | |
| "ndcg_at_10": 0.60056, | |
| "ndcg_at_20": 0.63495, | |
| "ndcg_at_50": 0.69554, | |
| "ndcg_at_100": 0.71726, | |
| "map_at_1": 0.33339, | |
| "map_at_3": 0.4226, | |
| "map_at_5": 0.45105, | |
| "map_at_10": 0.49113, | |
| "map_at_20": 0.52543, | |
| "map_at_50": 0.5505, | |
| "map_at_100": 0.55603, | |
| "recall_at_1": 0.33339, | |
| "recall_at_3": 0.46525, | |
| "recall_at_5": 0.5196, | |
| "recall_at_10": 0.60514, | |
| "recall_at_20": 0.70862, | |
| "recall_at_50": 0.89775, | |
| "recall_at_100": 0.97606, | |
| "precision_at_1": 0.65278, | |
| "precision_at_3": 0.40741, | |
| "precision_at_5": 0.31667, | |
| "precision_at_10": 0.22361, | |
| "precision_at_20": 0.14861, | |
| "precision_at_50": 0.08111, | |
| "precision_at_100": 0.04528, | |
| "mrr_at_1": 0.6527777777777778, | |
| "mrr_at_3": 0.7037037037037037, | |
| "mrr_at_5": 0.7203703703703704, | |
| "mrr_at_10": 0.7238977072310406, | |
| "mrr_at_20": 0.7247657627865962, | |
| "mrr_at_50": 0.7270465535079976, | |
| "mrr_at_100": 0.7274945821818327, | |
| "naucs_at_1_max": 0.22507029768221098, | |
| "naucs_at_1_std": 0.13788316928206162, | |
| "naucs_at_1_diff1": 0.5995848183367006, | |
| "naucs_at_3_max": 0.10374741818519737, | |
| "naucs_at_3_std": 0.1212402799460214, | |
| "naucs_at_3_diff1": 0.032982270408976666, | |
| "naucs_at_5_max": 0.036545011697732825, | |
| "naucs_at_5_std": 0.04976840467312129, | |
| "naucs_at_5_diff1": -0.06474423852856963, | |
| "naucs_at_10_max": -0.012180820225981331, | |
| "naucs_at_10_std": 0.007819844618087223, | |
| "naucs_at_10_diff1": -0.11570751534998236, | |
| "naucs_at_20_max": -0.05746500182397195, | |
| "naucs_at_20_std": -0.02111513643125022, | |
| "naucs_at_20_diff1": -0.12602864520641527, | |
| "naucs_at_50_max": -0.1124769198431645, | |
| "naucs_at_50_std": -0.07628706085741649, | |
| "naucs_at_50_diff1": -0.19607255004920401, | |
| "naucs_at_100_max": -0.11390067714346258, | |
| "naucs_at_100_std": -0.09427043796285493, | |
| "naucs_at_100_diff1": -0.25292751670835045 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.46875, | |
| "ndcg_at_3": 0.48885, | |
| "ndcg_at_5": 0.51496, | |
| "ndcg_at_10": 0.54548, | |
| "ndcg_at_20": 0.57449, | |
| "ndcg_at_50": 0.60556, | |
| "ndcg_at_100": 0.62078, | |
| "map_at_1": 0.27801, | |
| "map_at_3": 0.38689, | |
| "map_at_5": 0.42623, | |
| "map_at_10": 0.45364, | |
| "map_at_20": 0.46929, | |
| "map_at_50": 0.47926, | |
| "map_at_100": 0.48215, | |
| "recall_at_1": 0.27801, | |
| "recall_at_3": 0.46662, | |
| "recall_at_5": 0.55996, | |
| "recall_at_10": 0.65464, | |
| "recall_at_20": 0.73468, | |
| "recall_at_50": 0.83743, | |
| "recall_at_100": 0.90162, | |
| "precision_at_1": 0.46875, | |
| "precision_at_3": 0.30781, | |
| "precision_at_5": 0.24, | |
| "precision_at_10": 0.15109, | |
| "precision_at_20": 0.09297, | |
| "precision_at_50": 0.04666, | |
| "precision_at_100": 0.02609, | |
| "mrr_at_1": 0.46875, | |
| "mrr_at_3": 0.5653645833333331, | |
| "mrr_at_5": 0.5828645833333328, | |
| "mrr_at_10": 0.5928670634920632, | |
| "mrr_at_20": 0.5961635393061956, | |
| "mrr_at_50": 0.5979642273664314, | |
| "mrr_at_100": 0.598388641842729, | |
| "naucs_at_1_max": 0.25717698422505025, | |
| "naucs_at_1_std": -0.10478191029284567, | |
| "naucs_at_1_diff1": 0.4072573005477985, | |
| "naucs_at_3_max": 0.2659990641306254, | |
| "naucs_at_3_std": 0.04479561155056106, | |
| "naucs_at_3_diff1": 0.11139438643390147, | |
| "naucs_at_5_max": 0.2170152015957716, | |
| "naucs_at_5_std": 0.06816504281074703, | |
| "naucs_at_5_diff1": 0.03610018938984803, | |
| "naucs_at_10_max": 0.23222160900815464, | |
| "naucs_at_10_std": 0.1310209006859444, | |
| "naucs_at_10_diff1": -0.038456197207725154, | |
| "naucs_at_20_max": 0.27497446589778796, | |
| "naucs_at_20_std": 0.208324896139066, | |
| "naucs_at_20_diff1": -0.08762814336480476, | |
| "naucs_at_50_max": 0.27441764448304734, | |
| "naucs_at_50_std": 0.2529300579928729, | |
| "naucs_at_50_diff1": -0.11931330945117197, | |
| "naucs_at_100_max": 0.26643076503646324, | |
| "naucs_at_100_std": 0.23283573973890204, | |
| "naucs_at_100_diff1": -0.09996794144423607 | |
| }, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.64224, | |
| "ndcg_at_3": 0.60064, | |
| "ndcg_at_5": 0.55019, | |
| "ndcg_at_10": 0.52755, | |
| "ndcg_at_20": 0.55431, | |
| "ndcg_at_50": 0.62193, | |
| "ndcg_at_100": 0.66273, | |
| "map_at_1": 0.12328, | |
| "map_at_3": 0.20176, | |
| "map_at_5": 0.23781, | |
| "map_at_10": 0.29317, | |
| "map_at_20": 0.3496, | |
| "map_at_50": 0.40329, | |
| "map_at_100": 0.42827, | |
| "recall_at_1": 0.12328, | |
| "recall_at_3": 0.22334, | |
| "recall_at_5": 0.27732, | |
| "recall_at_10": 0.39122, | |
| "recall_at_20": 0.54233, | |
| "recall_at_50": 0.74043, | |
| "recall_at_100": 0.86408, | |
| "precision_at_1": 0.64224, | |
| "precision_at_3": 0.54023, | |
| "precision_at_5": 0.45345, | |
| "precision_at_10": 0.36509, | |
| "precision_at_20": 0.28384, | |
| "precision_at_50": 0.18336, | |
| "precision_at_100": 0.11944, | |
| "mrr_at_1": 0.6422413793103449, | |
| "mrr_at_3": 0.726293103448276, | |
| "mrr_at_5": 0.7344827586206897, | |
| "mrr_at_10": 0.7423662424740011, | |
| "mrr_at_20": 0.7454521368427901, | |
| "mrr_at_50": 0.7464683101612821, | |
| "mrr_at_100": 0.7466140292051692, | |
| "naucs_at_1_max": 0.04669677333287125, | |
| "naucs_at_1_std": -0.047276436301337176, | |
| "naucs_at_1_diff1": 0.20998438017074644, | |
| "naucs_at_3_max": 0.1207746945432274, | |
| "naucs_at_3_std": 0.11705226278367024, | |
| "naucs_at_3_diff1": -0.0659769870275157, | |
| "naucs_at_5_max": 0.17489805487159318, | |
| "naucs_at_5_std": 0.15087482671660668, | |
| "naucs_at_5_diff1": -0.10486886234938332, | |
| "naucs_at_10_max": 0.13971495080647375, | |
| "naucs_at_10_std": 0.12342403085916763, | |
| "naucs_at_10_diff1": -0.11904046578732998, | |
| "naucs_at_20_max": 0.1262352504070948, | |
| "naucs_at_20_std": 0.1870795410571109, | |
| "naucs_at_20_diff1": -0.09021505632729916, | |
| "naucs_at_50_max": 0.08609145496415921, | |
| "naucs_at_50_std": 0.19880949579195997, | |
| "naucs_at_50_diff1": -0.10141550865230196, | |
| "naucs_at_100_max": 0.029463316752851777, | |
| "naucs_at_100_std": 0.1254397237152911, | |
| "naucs_at_100_diff1": -0.10340073450123488 | |
| }, | |
| "vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.78, | |
| "ndcg_at_3": 0.83717, | |
| "ndcg_at_5": 0.8493, | |
| "ndcg_at_10": 0.85579, | |
| "ndcg_at_20": 0.86416, | |
| "ndcg_at_50": 0.86978, | |
| "ndcg_at_100": 0.8714, | |
| "map_at_1": 0.78, | |
| "map_at_3": 0.82367, | |
| "map_at_5": 0.83027, | |
| "map_at_10": 0.83296, | |
| "map_at_20": 0.83513, | |
| "map_at_50": 0.83606, | |
| "map_at_100": 0.8362, | |
| "recall_at_1": 0.78, | |
| "recall_at_3": 0.876, | |
| "recall_at_5": 0.906, | |
| "recall_at_10": 0.926, | |
| "recall_at_20": 0.96, | |
| "recall_at_50": 0.988, | |
| "recall_at_100": 0.998, | |
| "precision_at_1": 0.78, | |
| "precision_at_3": 0.292, | |
| "precision_at_5": 0.1812, | |
| "precision_at_10": 0.0926, | |
| "precision_at_20": 0.048, | |
| "precision_at_50": 0.01976, | |
| "precision_at_100": 0.00998, | |
| "mrr_at_1": 0.78, | |
| "mrr_at_3": 0.8236666666666668, | |
| "mrr_at_5": 0.8302666666666666, | |
| "mrr_at_10": 0.8329571428571427, | |
| "mrr_at_20": 0.8351287569413728, | |
| "mrr_at_50": 0.8360576689817807, | |
| "mrr_at_100": 0.8362028700500819, | |
| "naucs_at_1_max": 0.7913757128946993, | |
| "naucs_at_1_std": -0.6034404414151248, | |
| "naucs_at_1_diff1": 0.9052951268141137, | |
| "naucs_at_3_max": 0.7991254905596246, | |
| "naucs_at_3_std": -0.5793856802941784, | |
| "naucs_at_3_diff1": 0.8585874972961292, | |
| "naucs_at_5_max": 0.7909291376124926, | |
| "naucs_at_5_std": -0.4963148379919354, | |
| "naucs_at_5_diff1": 0.8356576673222494, | |
| "naucs_at_10_max": 0.8344689227042192, | |
| "naucs_at_10_std": -0.5341433870845648, | |
| "naucs_at_10_diff1": 0.8497867615514662, | |
| "naucs_at_20_max": 0.8126984126984111, | |
| "naucs_at_20_std": -0.4176003734827309, | |
| "naucs_at_20_diff1": 0.8263305322128826, | |
| "naucs_at_50_max": 0.9074074074074128, | |
| "naucs_at_50_std": -0.04474011826952445, | |
| "naucs_at_50_diff1": 0.860333022097726, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.45455, | |
| "ndcg_at_3": 0.53012, | |
| "ndcg_at_5": 0.55569, | |
| "ndcg_at_10": 0.57792, | |
| "ndcg_at_20": 0.59293, | |
| "ndcg_at_50": 0.60814, | |
| "ndcg_at_100": 0.61994, | |
| "map_at_1": 0.45455, | |
| "map_at_3": 0.51109, | |
| "map_at_5": 0.52528, | |
| "map_at_10": 0.53447, | |
| "map_at_20": 0.53882, | |
| "map_at_50": 0.54136, | |
| "map_at_100": 0.54249, | |
| "recall_at_1": 0.45455, | |
| "recall_at_3": 0.58537, | |
| "recall_at_5": 0.64745, | |
| "recall_at_10": 0.71619, | |
| "recall_at_20": 0.77384, | |
| "recall_at_50": 0.84922, | |
| "recall_at_100": 0.92018, | |
| "precision_at_1": 0.45455, | |
| "precision_at_3": 0.19512, | |
| "precision_at_5": 0.12949, | |
| "precision_at_10": 0.07162, | |
| "precision_at_20": 0.03869, | |
| "precision_at_50": 0.01698, | |
| "precision_at_100": 0.0092, | |
| "mrr_at_1": 0.45454545454545453, | |
| "mrr_at_3": 0.5110864745011089, | |
| "mrr_at_5": 0.5252771618625277, | |
| "mrr_at_10": 0.5344683771513038, | |
| "mrr_at_20": 0.5388215531769072, | |
| "mrr_at_50": 0.5413611772588918, | |
| "mrr_at_100": 0.5424869155538622, | |
| "naucs_at_1_max": 0.4075821857009155, | |
| "naucs_at_1_std": -0.310949225760234, | |
| "naucs_at_1_diff1": 0.7407514800732501, | |
| "naucs_at_3_max": 0.46743175097720985, | |
| "naucs_at_3_std": -0.1986126839848541, | |
| "naucs_at_3_diff1": 0.6505296132803555, | |
| "naucs_at_5_max": 0.5364564428320254, | |
| "naucs_at_5_std": -0.14408657067489153, | |
| "naucs_at_5_diff1": 0.6322410022842855, | |
| "naucs_at_10_max": 0.5825512151708625, | |
| "naucs_at_10_std": -0.07725749814895451, | |
| "naucs_at_10_diff1": 0.6332882465209557, | |
| "naucs_at_20_max": 0.5813970577815284, | |
| "naucs_at_20_std": -0.02416200306109636, | |
| "naucs_at_20_diff1": 0.5952214798946482, | |
| "naucs_at_50_max": 0.6298047922819098, | |
| "naucs_at_50_std": 0.21852001263213766, | |
| "naucs_at_50_diff1": 0.5810079368486553, | |
| "naucs_at_100_max": 0.6564226194686283, | |
| "naucs_at_100_std": 0.46804642848156325, | |
| "naucs_at_100_diff1": 0.49830945198923904 | |
| }, | |
| "vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.7753, | |
| "ndcg_at_3": 0.84285, | |
| "ndcg_at_5": 0.85383, | |
| "ndcg_at_10": 0.86362, | |
| "ndcg_at_20": 0.86829, | |
| "ndcg_at_50": 0.87162, | |
| "ndcg_at_100": 0.87297, | |
| "map_at_1": 0.7753, | |
| "map_at_3": 0.82692, | |
| "map_at_5": 0.8331, | |
| "map_at_10": 0.83712, | |
| "map_at_20": 0.83843, | |
| "map_at_50": 0.83902, | |
| "map_at_100": 0.83915, | |
| "recall_at_1": 0.7753, | |
| "recall_at_3": 0.88866, | |
| "recall_at_5": 0.91498, | |
| "recall_at_10": 0.94534, | |
| "recall_at_20": 0.96356, | |
| "recall_at_50": 0.97976, | |
| "recall_at_100": 0.98785, | |
| "precision_at_1": 0.7753, | |
| "precision_at_3": 0.29622, | |
| "precision_at_5": 0.183, | |
| "precision_at_10": 0.09453, | |
| "precision_at_20": 0.04818, | |
| "precision_at_50": 0.0196, | |
| "precision_at_100": 0.00988, | |
| "mrr_at_1": 0.7753036437246964, | |
| "mrr_at_3": 0.8269230769230768, | |
| "mrr_at_5": 0.8330971659919026, | |
| "mrr_at_10": 0.8371192404087138, | |
| "mrr_at_20": 0.8384322229490562, | |
| "mrr_at_50": 0.8390209545984961, | |
| "mrr_at_100": 0.8391494570651827, | |
| "naucs_at_1_max": 0.42955277741228204, | |
| "naucs_at_1_std": -0.413540459458001, | |
| "naucs_at_1_diff1": 0.8615855669987156, | |
| "naucs_at_3_max": 0.46912857131917574, | |
| "naucs_at_3_std": -0.3714926257299717, | |
| "naucs_at_3_diff1": 0.82680032161878, | |
| "naucs_at_5_max": 0.6392482503231406, | |
| "naucs_at_5_std": -0.2299842899342104, | |
| "naucs_at_5_diff1": 0.8417184248940351, | |
| "naucs_at_10_max": 0.7086238741706962, | |
| "naucs_at_10_std": -0.0498141562896836, | |
| "naucs_at_10_diff1": 0.837619456912457, | |
| "naucs_at_20_max": 0.6570477401161009, | |
| "naucs_at_20_std": 0.0009204781615544056, | |
| "naucs_at_20_diff1": 0.7936258020327029, | |
| "naucs_at_50_max": 0.774779293140126, | |
| "naucs_at_50_std": 0.35856346999101907, | |
| "naucs_at_50_diff1": 0.7547641636196891, | |
| "naucs_at_100_max": 0.7116997912767351, | |
| "naucs_at_100_std": 0.3194376069347773, | |
| "naucs_at_100_diff1": 0.7833756960419608 | |
| }, | |
| "vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.87143, | |
| "ndcg_at_3": 0.91866, | |
| "ndcg_at_5": 0.92758, | |
| "ndcg_at_10": 0.92996, | |
| "ndcg_at_20": 0.9336, | |
| "ndcg_at_50": 0.93499, | |
| "ndcg_at_100": 0.93499, | |
| "map_at_1": 0.87143, | |
| "map_at_3": 0.90774, | |
| "map_at_5": 0.91274, | |
| "map_at_10": 0.91376, | |
| "map_at_20": 0.91476, | |
| "map_at_50": 0.91497, | |
| "map_at_100": 0.91497, | |
| "recall_at_1": 0.87143, | |
| "recall_at_3": 0.95, | |
| "recall_at_5": 0.97143, | |
| "recall_at_10": 0.97857, | |
| "recall_at_20": 0.99286, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.87143, | |
| "precision_at_3": 0.31667, | |
| "precision_at_5": 0.19429, | |
| "precision_at_10": 0.09786, | |
| "precision_at_20": 0.04964, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.8714285714285714, | |
| "mrr_at_3": 0.9077380952380953, | |
| "mrr_at_5": 0.9127380952380953, | |
| "mrr_at_10": 0.9137585034013606, | |
| "mrr_at_20": 0.9147645211930926, | |
| "mrr_at_50": 0.9149716040262259, | |
| "mrr_at_100": 0.9149716040262259, | |
| "naucs_at_1_max": 0.7871300352601777, | |
| "naucs_at_1_std": 0.20583128539373943, | |
| "naucs_at_1_diff1": 0.9197964526124592, | |
| "naucs_at_3_max": 0.9229691876750705, | |
| "naucs_at_3_std": 0.460784313725494, | |
| "naucs_at_3_diff1": 0.9626517273576086, | |
| "naucs_at_5_max": 0.9673202614379075, | |
| "naucs_at_5_std": 0.562791783380022, | |
| "naucs_at_5_diff1": 0.934640522875815, | |
| "naucs_at_10_max": 0.9782135076252753, | |
| "naucs_at_10_std": 0.7584811702458847, | |
| "naucs_at_10_diff1": 0.9128540305010935, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 0.8692810457516309, | |
| "naucs_at_50_max": 1.0, | |
| "naucs_at_50_std": 1.0, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.55346, | |
| "ndcg_at_3": 0.66654, | |
| "ndcg_at_5": 0.69922, | |
| "ndcg_at_10": 0.72683, | |
| "ndcg_at_20": 0.73868, | |
| "ndcg_at_50": 0.74458, | |
| "ndcg_at_100": 0.74555, | |
| "map_at_1": 0.55346, | |
| "map_at_3": 0.63902, | |
| "map_at_5": 0.65725, | |
| "map_at_10": 0.66893, | |
| "map_at_20": 0.67236, | |
| "map_at_50": 0.6733, | |
| "map_at_100": 0.67338, | |
| "recall_at_1": 0.55346, | |
| "recall_at_3": 0.74605, | |
| "recall_at_5": 0.82503, | |
| "recall_at_10": 0.90887, | |
| "recall_at_20": 0.95443, | |
| "recall_at_50": 0.9842, | |
| "recall_at_100": 0.99028, | |
| "precision_at_1": 0.55346, | |
| "precision_at_3": 0.24868, | |
| "precision_at_5": 0.16501, | |
| "precision_at_10": 0.09089, | |
| "precision_at_20": 0.04772, | |
| "precision_at_50": 0.01968, | |
| "precision_at_100": 0.0099, | |
| "mrr_at_1": 0.5552855407047388, | |
| "mrr_at_3": 0.6395301741595798, | |
| "mrr_at_5": 0.6578473066018649, | |
| "mrr_at_10": 0.6696035603386765, | |
| "mrr_at_20": 0.6729832012543134, | |
| "mrr_at_50": 0.6739181612518058, | |
| "mrr_at_100": 0.6739995114642742, | |
| "naucs_at_1_max": 0.4481578507912886, | |
| "naucs_at_1_std": -0.28229697462188064, | |
| "naucs_at_1_diff1": 0.6794797414689293, | |
| "naucs_at_3_max": 0.521452354349859, | |
| "naucs_at_3_std": -0.2525394663950513, | |
| "naucs_at_3_diff1": 0.5641368811967125, | |
| "naucs_at_5_max": 0.5522320601024093, | |
| "naucs_at_5_std": -0.23449944203419384, | |
| "naucs_at_5_diff1": 0.5550996108687463, | |
| "naucs_at_10_max": 0.6709947806703614, | |
| "naucs_at_10_std": -0.018864423203824147, | |
| "naucs_at_10_diff1": 0.5147848561655853, | |
| "naucs_at_20_max": 0.6719932800208073, | |
| "naucs_at_20_std": 0.18814578113635355, | |
| "naucs_at_20_diff1": 0.4774949408808388, | |
| "naucs_at_50_max": 0.710665729775863, | |
| "naucs_at_50_std": 0.409805723201301, | |
| "naucs_at_50_diff1": 0.4920058027972864, | |
| "naucs_at_100_max": 0.6822698211447887, | |
| "naucs_at_100_std": 0.3946630393669552, | |
| "naucs_at_100_diff1": 0.5232939215033167 | |
| }, | |
| "vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.59, | |
| "ndcg_at_3": 0.76142, | |
| "ndcg_at_5": 0.78683, | |
| "ndcg_at_10": 0.8064, | |
| "ndcg_at_20": 0.8064, | |
| "ndcg_at_50": 0.8064, | |
| "ndcg_at_100": 0.80812, | |
| "map_at_1": 0.59, | |
| "map_at_3": 0.72333, | |
| "map_at_5": 0.73783, | |
| "map_at_10": 0.74602, | |
| "map_at_20": 0.74602, | |
| "map_at_50": 0.74602, | |
| "map_at_100": 0.74621, | |
| "recall_at_1": 0.59, | |
| "recall_at_3": 0.87, | |
| "recall_at_5": 0.93, | |
| "recall_at_10": 0.99, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 0.99, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.59, | |
| "precision_at_3": 0.29, | |
| "precision_at_5": 0.186, | |
| "precision_at_10": 0.099, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.0198, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.59, | |
| "mrr_at_3": 0.7233333333333333, | |
| "mrr_at_5": 0.7378333333333333, | |
| "mrr_at_10": 0.7460238095238095, | |
| "mrr_at_20": 0.7460238095238095, | |
| "mrr_at_50": 0.7460238095238095, | |
| "mrr_at_100": 0.7462056277056278, | |
| "naucs_at_1_max": 0.14863456938676678, | |
| "naucs_at_1_std": -0.41319500491584377, | |
| "naucs_at_1_diff1": 0.6677623373002481, | |
| "naucs_at_3_max": 0.34644153151151447, | |
| "naucs_at_3_std": -0.3760275494334602, | |
| "naucs_at_3_diff1": 0.5028141894393838, | |
| "naucs_at_5_max": 0.29825263438709065, | |
| "naucs_at_5_std": -0.40456182472989133, | |
| "naucs_at_5_diff1": 0.434507136187809, | |
| "naucs_at_10_max": -0.1713352007469681, | |
| "naucs_at_10_std": -0.5634920634920767, | |
| "naucs_at_10_diff1": -1.1517273576097316, | |
| "naucs_at_20_max": -0.1713352007469681, | |
| "naucs_at_20_std": -0.5634920634920767, | |
| "naucs_at_20_diff1": -1.1517273576097316, | |
| "naucs_at_50_max": -0.17133520074697067, | |
| "naucs_at_50_std": -0.5634920634920583, | |
| "naucs_at_50_diff1": -1.1517273576096694, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 0.87, | |
| "ndcg_at_3": 0.94047, | |
| "ndcg_at_5": 0.94478, | |
| "ndcg_at_10": 0.94478, | |
| "ndcg_at_20": 0.94478, | |
| "ndcg_at_50": 0.94478, | |
| "ndcg_at_100": 0.94478, | |
| "map_at_1": 0.87, | |
| "map_at_3": 0.92333, | |
| "map_at_5": 0.92583, | |
| "map_at_10": 0.92583, | |
| "map_at_20": 0.92583, | |
| "map_at_50": 0.92583, | |
| "map_at_100": 0.92583, | |
| "recall_at_1": 0.87, | |
| "recall_at_3": 0.99, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.87, | |
| "precision_at_3": 0.33, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.87, | |
| "mrr_at_3": 0.9233333333333333, | |
| "mrr_at_5": 0.9258333333333333, | |
| "mrr_at_10": 0.9258333333333333, | |
| "mrr_at_20": 0.9258333333333333, | |
| "mrr_at_50": 0.9258333333333333, | |
| "mrr_at_100": 0.9258333333333333, | |
| "naucs_at_1_max": 0.37958231504110235, | |
| "naucs_at_1_std": 0.006480041472265402, | |
| "naucs_at_1_diff1": 0.9063171147152481, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 0.7222222222222157, | |
| "naucs_at_3_diff1": 0.8692810457516356, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.85, | |
| "ndcg_at_3": 0.89155, | |
| "ndcg_at_5": 0.89542, | |
| "ndcg_at_10": 0.91169, | |
| "ndcg_at_20": 0.91448, | |
| "ndcg_at_50": 0.91661, | |
| "ndcg_at_100": 0.91661, | |
| "map_at_1": 0.85, | |
| "map_at_3": 0.88167, | |
| "map_at_5": 0.88367, | |
| "map_at_10": 0.89044, | |
| "map_at_20": 0.89135, | |
| "map_at_50": 0.89175, | |
| "map_at_100": 0.89175, | |
| "recall_at_1": 0.85, | |
| "recall_at_3": 0.92, | |
| "recall_at_5": 0.93, | |
| "recall_at_10": 0.98, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.85, | |
| "precision_at_3": 0.30667, | |
| "precision_at_5": 0.186, | |
| "precision_at_10": 0.098, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.85, | |
| "mrr_at_3": 0.8816666666666667, | |
| "mrr_at_5": 0.8836666666666667, | |
| "mrr_at_10": 0.8904404761904763, | |
| "mrr_at_20": 0.8913495670995673, | |
| "mrr_at_50": 0.8917495670995673, | |
| "mrr_at_100": 0.8917495670995673, | |
| "naucs_at_1_max": 0.36173120728929387, | |
| "naucs_at_1_std": -0.9650178978197211, | |
| "naucs_at_1_diff1": 0.9199479336153596, | |
| "naucs_at_3_max": 0.4982492997198882, | |
| "naucs_at_3_std": -1.0291783380018664, | |
| "naucs_at_3_diff1": 0.932598039215688, | |
| "naucs_at_5_max": 0.5939042283580102, | |
| "naucs_at_5_std": -1.0116713352007418, | |
| "naucs_at_5_diff1": 0.96265172735761, | |
| "naucs_at_10_max": 0.21825396825397442, | |
| "naucs_at_10_std": -1.7399626517273414, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": -1.7399626517273863, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.9, | |
| "ndcg_at_3": 0.96047, | |
| "ndcg_at_5": 0.96047, | |
| "ndcg_at_10": 0.96047, | |
| "ndcg_at_20": 0.96047, | |
| "ndcg_at_50": 0.96047, | |
| "ndcg_at_100": 0.96047, | |
| "map_at_1": 0.9, | |
| "map_at_3": 0.94667, | |
| "map_at_5": 0.94667, | |
| "map_at_10": 0.94667, | |
| "map_at_20": 0.94667, | |
| "map_at_50": 0.94667, | |
| "map_at_100": 0.94667, | |
| "recall_at_1": 0.9, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.9, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9, | |
| "mrr_at_3": 0.9466666666666668, | |
| "mrr_at_5": 0.9466666666666668, | |
| "mrr_at_10": 0.9466666666666668, | |
| "mrr_at_20": 0.9466666666666668, | |
| "mrr_at_50": 0.9466666666666668, | |
| "mrr_at_100": 0.9466666666666668, | |
| "naucs_at_1_max": 0.2663398692810457, | |
| "naucs_at_1_std": -0.1846405228758159, | |
| "naucs_at_1_diff1": 0.8835200746965451, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.87, | |
| "ndcg_at_3": 0.95071, | |
| "ndcg_at_5": 0.95071, | |
| "ndcg_at_10": 0.95071, | |
| "ndcg_at_20": 0.95071, | |
| "ndcg_at_50": 0.95071, | |
| "ndcg_at_100": 0.95071, | |
| "map_at_1": 0.87, | |
| "map_at_3": 0.93333, | |
| "map_at_5": 0.93333, | |
| "map_at_10": 0.93333, | |
| "map_at_20": 0.93333, | |
| "map_at_50": 0.93333, | |
| "map_at_100": 0.93333, | |
| "recall_at_1": 0.87, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.87, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.87, | |
| "mrr_at_3": 0.9333333333333332, | |
| "mrr_at_5": 0.9333333333333332, | |
| "mrr_at_10": 0.9333333333333332, | |
| "mrr_at_20": 0.9333333333333332, | |
| "mrr_at_50": 0.9333333333333332, | |
| "mrr_at_100": 0.9333333333333332, | |
| "naucs_at_1_max": 0.640339183885063, | |
| "naucs_at_1_std": -0.4568984670073326, | |
| "naucs_at_1_diff1": 0.9339035769828927, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| } | |
| } | |
| } |