{ "metadata": { "timestamp": "2025-01-31T14:00:16.349261", "vidore_benchmark_version": "4.0.3.dev20+g2d72668" }, "metrics": { "vidore/restaurant_esg_reports_beir": {"ndcg_at_1": 0.64103, "ndcg_at_3": 0.67538, "ndcg_at_5": 0.68397, "ndcg_at_10": 0.71199, "ndcg_at_20": 0.73099, "ndcg_at_50": 0.75581, "ndcg_at_100": 0.76163, "map_at_1": 0.43942, "map_at_3": 0.58034, "map_at_5": 0.6105, "map_at_10": 0.63715, "map_at_20": 0.64674, "map_at_50": 0.65463, "map_at_100": 0.65653, "recall_at_1": 0.43942, "recall_at_3": 0.6766, "recall_at_5": 0.73558, "recall_at_10": 0.81426, "recall_at_20": 0.87567, "recall_at_50": 0.95485, "recall_at_100": 0.981, "precision_at_1": 0.65385, "precision_at_3": 0.39744, "precision_at_5": 0.27692, "precision_at_10": 0.16538, "precision_at_20": 0.09423, "precision_at_50": 0.04346, "precision_at_100": 0.02308, "mrr_at_1": 0.6730769230769231, "mrr_at_3": 0.7467948717948718, "mrr_at_5": 0.7592948717948718, "mrr_at_10": 0.7647893772893772, "mrr_at_20": 0.7647893772893772, "mrr_at_50": 0.76724122651542, "mrr_at_100": 0.76724122651542, "naucs_at_1_max": 0.10629249714250551, "naucs_at_1_std": 0.03656703481974433, "naucs_at_1_diff1": 0.6471348946136236, "naucs_at_3_max": -0.04531273341377843, "naucs_at_3_std": 0.00041838369923711004, "naucs_at_3_diff1": -0.10904002079315368, "naucs_at_5_max": -0.11437346877355879, "naucs_at_5_std": -0.03247506128676773, "naucs_at_5_diff1": -0.23163739837531055, "naucs_at_10_max": -0.17966299098993738, "naucs_at_10_std": -0.08845607083585832, "naucs_at_10_diff1": -0.3668906677620788, "naucs_at_20_max": -0.1623997659302278, "naucs_at_20_std": -0.05328921373458377, "naucs_at_20_diff1": -0.4139154609733948, "naucs_at_50_max": -0.15835976853630981, "naucs_at_50_std": -0.042923187520787366, "naucs_at_50_diff1": -0.45626822378009313, "naucs_at_100_max": -0.14433851613678755, "naucs_at_100_std": -0.03148299510892002, "naucs_at_100_diff1": -0.44576138476398025}, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.60345, "ndcg_at_3": 0.59658, "ndcg_at_5": 0.56521, "ndcg_at_10": 0.55055, "ndcg_at_20": 0.57899, "ndcg_at_50": 0.65246, "ndcg_at_100": 0.68858, "map_at_1": 0.08192, "map_at_3": 0.18869, "map_at_5": 0.23861, "map_at_10": 0.30685, "map_at_20": 0.36485, "map_at_50": 0.42784, "map_at_100": 0.45362, "recall_at_1": 0.08192, "recall_at_3": 0.23525, "recall_at_5": 0.31079, "recall_at_10": 0.43833, "recall_at_20": 0.58306, "recall_at_50": 0.78859, "recall_at_100": 0.90965, "precision_at_1": 0.60345, "precision_at_3": 0.5546, "precision_at_5": 0.49138, "precision_at_10": 0.39914, "precision_at_20": 0.30539, "precision_at_50": 0.1981, "precision_at_100": 0.12647, "mrr_at_1": 0.5948275862068966, "mrr_at_3": 0.7140804597701149, "mrr_at_5": 0.7311063218390805, "mrr_at_10": 0.7372383004926109, "mrr_at_20": 0.7380691285134744, "mrr_at_50": 0.738889335917265, "mrr_at_100": 0.738889335917265, "naucs_at_1_max": 0.009301400086149124, "naucs_at_1_std": -0.018176205759631064, "naucs_at_1_diff1": 0.06589141538556753, "naucs_at_3_max": 0.03608007053002765, "naucs_at_3_std": 0.045332981794521165, "naucs_at_3_diff1": 0.0027274204838523214, "naucs_at_5_max": 0.06351874275538175, "naucs_at_5_std": 0.09161670694703962, "naucs_at_5_diff1": 0.030104357247264922, "naucs_at_10_max": 0.09045732284339944, "naucs_at_10_std": 0.11937912992266166, "naucs_at_10_diff1": -0.007723827231040398, "naucs_at_20_max": 0.04097490582062997, "naucs_at_20_std": 0.11121740670187244, "naucs_at_20_diff1": -0.04080919911505186, "naucs_at_50_max": -0.04431654376754045, "naucs_at_50_std": 0.1075378398002067, "naucs_at_50_diff1": -0.0408768243788732, "naucs_at_100_max": -0.08194156783211, "naucs_at_100_std": 0.08152221906964778, "naucs_at_100_diff1": -0.002462405639259035 }, "vidore/synthetic_axa_filtered_v1.0_multilingual": { "ndcg_at_1": 0.44444, "ndcg_at_3": 0.52927, "ndcg_at_5": 0.53193, "ndcg_at_10": 0.55273, "ndcg_at_20": 0.57791, "ndcg_at_50": 0.62959, "ndcg_at_100": 0.64947, "map_at_1": 0.19923, "map_at_3": 0.31897, "map_at_5": 0.3681, "map_at_10": 0.4284, "map_at_20": 0.45565, "map_at_50": 0.47849, "map_at_100": 0.48445, "recall_at_1": 0.19923, "recall_at_3": 0.43016, "recall_at_5": 0.51283, "recall_at_10": 0.63101, "recall_at_20": 0.7193, "recall_at_50": 0.86749, "recall_at_100": 0.93786, "precision_at_1": 0.44444, "precision_at_3": 0.39815, "precision_at_5": 0.33611, "precision_at_10": 0.25, "precision_at_20": 0.15347, "precision_at_50": 0.08167, "precision_at_100": 0.04556, "mrr_at_1": 0.4444444444444444, "mrr_at_3": 0.6041666666666666, "mrr_at_5": 0.6111111111111112, "mrr_at_10": 0.6134259259259259, "mrr_at_20": 0.6134259259259259, "mrr_at_50": 0.6156871267846877, "mrr_at_100": 0.6160388005738615, "naucs_at_1_max": -0.007676494689249098, "naucs_at_1_std": 0.15632886231764911, "naucs_at_1_diff1": 0.3454147669831923, "naucs_at_3_max": -0.21520431732796824, "naucs_at_3_std": -0.039307896873910286, "naucs_at_3_diff1": 0.07075823324379214, "naucs_at_5_max": -0.23427920743418434, "naucs_at_5_std": -0.04955213973003769, "naucs_at_5_diff1": 0.03587555672187707, "naucs_at_10_max": -0.32076687527735803, "naucs_at_10_std": -0.08584531062590048, "naucs_at_10_diff1": -0.13920437910700148, "naucs_at_20_max": -0.3500838034005256, "naucs_at_20_std": -0.11679539412969686, "naucs_at_20_diff1": -0.2051894549829469, "naucs_at_50_max": -0.4102497000902005, "naucs_at_50_std": -0.1409315271751862, "naucs_at_50_diff1": -0.16443764913412845, "naucs_at_100_max": -0.43730180953377223, "naucs_at_100_std": -0.17380745870110292, "naucs_at_100_diff1": -0.19442949153976513 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.61875, "ndcg_at_3": 0.61406, "ndcg_at_5": 0.63644, "ndcg_at_10": 0.67591, "ndcg_at_20": 0.70178, "ndcg_at_50": 0.72212, "ndcg_at_100": 0.7297, "map_at_1": 0.38142, "map_at_3": 0.50057, "map_at_5": 0.54282, "map_at_10": 0.58278, "map_at_20": 0.59965, "map_at_50": 0.60789, "map_at_100": 0.60942, "recall_at_1": 0.38142, "recall_at_3": 0.55686, "recall_at_5": 0.65567, "recall_at_10": 0.77349, "recall_at_20": 0.84542, "recall_at_50": 0.902, "recall_at_100": 0.93314, "precision_at_1": 0.61875, "precision_at_3": 0.38333, "precision_at_5": 0.28875, "precision_at_10": 0.18875, "precision_at_20": 0.11063, "precision_at_50": 0.052, "precision_at_100": 0.02756, "mrr_at_1": 0.6, "mrr_at_3": 0.6947916666666667, "mrr_at_5": 0.7104166666666667, "mrr_at_10": 0.7197544642857143, "mrr_at_20": 0.7207690746753247, "mrr_at_50": 0.7216319385110503, "mrr_at_100": 0.7218020773999393, "naucs_at_1_max": 0.23570942089460648, "naucs_at_1_std": -0.0331808109585888, "naucs_at_1_diff1": 0.38829894385449926, "naucs_at_3_max": -0.02996939892755872, "naucs_at_3_std": -0.06455703309969216, "naucs_at_3_diff1": -0.10409043693090879, "naucs_at_5_max": -0.10452188626653831, "naucs_at_5_std": -0.052524231286355125, "naucs_at_5_diff1": -0.26201383842532566, "naucs_at_10_max": -0.18753439904110775, "naucs_at_10_std": -0.07594574430351896, "naucs_at_10_diff1": -0.3404263646481817, "naucs_at_20_max": -0.20945097879456231, "naucs_at_20_std": -0.04291331785264603, "naucs_at_20_diff1": -0.35991045256760157, "naucs_at_50_max": -0.25078519595626036, "naucs_at_50_std": -0.021452523948669356, "naucs_at_50_diff1": -0.3863780246795678, "naucs_at_100_max": -0.2723431611743009, "naucs_at_100_std": -0.031237471793638266, "naucs_at_100_diff1": -0.38639136112161093 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.57895, "ndcg_at_3": 0.57099, "ndcg_at_5": 0.57396, "ndcg_at_10": 0.6394, "ndcg_at_20": 0.67256, "ndcg_at_50": 0.70473, "ndcg_at_100": 0.71518, "map_at_1": 0.26466, "map_at_3": 0.41771, "map_at_5": 0.45328, "map_at_10": 0.515, "map_at_20": 0.53681, "map_at_50": 0.55475, "map_at_100": 0.55975, "recall_at_1": 0.26466, "recall_at_3": 0.50881, "recall_at_5": 0.57932, "recall_at_10": 0.77663, "recall_at_20": 0.8768, "recall_at_50": 0.96898, "recall_at_100": 1.0, "precision_at_1": 0.57895, "precision_at_3": 0.39766, "precision_at_5": 0.29123, "precision_at_10": 0.21053, "precision_at_20": 0.13246, "precision_at_50": 0.07123, "precision_at_100": 0.03895, "mrr_at_1": 0.5614035087719298, "mrr_at_3": 0.6637426900584795, "mrr_at_5": 0.6751461988304094, "mrr_at_10": 0.6886173767752715, "mrr_at_20": 0.6913087643350802, "mrr_at_50": 0.6925429103060682, "mrr_at_100": 0.6925429103060682, "naucs_at_1_max": -0.32399679124866293, "naucs_at_1_std": -0.3157496852536844, "naucs_at_1_diff1": 0.059217413521438306, "naucs_at_3_max": -0.16317940152105928, "naucs_at_3_std": -0.036766880845353206, "naucs_at_3_diff1": 0.13451653330920363, "naucs_at_5_max": -0.18872080560435078, "naucs_at_5_std": -0.08965899629213353, "naucs_at_5_diff1": 0.09733357617902798, "naucs_at_10_max": -0.21096127425243222, "naucs_at_10_std": -0.0492682492736064, "naucs_at_10_diff1": 0.06579043413332678, "naucs_at_20_max": -0.3151765999140926, "naucs_at_20_std": -0.11272929289257284, "naucs_at_20_diff1": -0.030784322032566636, "naucs_at_50_max": -0.29489025135650376, "naucs_at_50_std": -0.008523958191869246, "naucs_at_50_diff1": -0.11796822928434904, "naucs_at_100_max": -0.2661970083875134, "naucs_at_100_std": 0.05555820249689567, "naucs_at_100_diff1": -0.1271611921091859 }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.55702, "ndcg_at_3": 0.5542, "ndcg_at_5": 0.5738, "ndcg_at_10": 0.62953, "ndcg_at_20": 0.67065, "ndcg_at_50": 0.6964, "ndcg_at_100": 0.70872, "map_at_1": 0.25589, "map_at_3": 0.40701, "map_at_5": 0.45304, "map_at_10": 0.5087, "map_at_20": 0.53389, "map_at_50": 0.5501, "map_at_100": 0.55542, "recall_at_1": 0.25589, "recall_at_3": 0.49373, "recall_at_5": 0.59174, "recall_at_10": 0.76938, "recall_at_20": 0.89178, "recall_at_50": 0.9594, "recall_at_100": 1.0, "precision_at_1": 0.55702, "precision_at_3": 0.3845, "precision_at_5": 0.29649, "precision_at_10": 0.20746, "precision_at_20": 0.13487, "precision_at_50": 0.0707, "precision_at_100": 0.03895, "mrr_at_1": 0.5570175438596491, "mrr_at_3": 0.6571637426900585, "mrr_at_5": 0.6698830409356724, "mrr_at_10": 0.683766708437761, "mrr_at_20": 0.687599964534949, "mrr_at_50": 0.6879085010276961, "mrr_at_100": 0.6880572205275953, "naucs_at_1_max": -0.17166276074798967, "naucs_at_1_std": -0.20325235499471186, "naucs_at_1_diff1": 0.33001833715444423, "naucs_at_3_max": -0.04474887443402487, "naucs_at_3_std": -0.008560331928141176, "naucs_at_3_diff1": 0.1746255699902219, "naucs_at_5_max": -0.08458531884834858, "naucs_at_5_std": -0.06118868392476525, "naucs_at_5_diff1": 0.13235525060593817, "naucs_at_10_max": -0.12880215182381638, "naucs_at_10_std": -0.08311726109553826, "naucs_at_10_diff1": 0.11900519462159719, "naucs_at_20_max": -0.18439650375622735, "naucs_at_20_std": -0.051821500134533655, "naucs_at_20_diff1": -0.015123572562768748, "naucs_at_50_max": -0.20702547660326365, "naucs_at_50_std": 0.02211997502899773, "naucs_at_50_diff1": -0.1539679823951051, "naucs_at_100_max": -0.1731060459048004, "naucs_at_100_std": 0.1019715434458989, "naucs_at_100_diff1": -0.1841288831466728 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.57812, "ndcg_at_3": 0.58744, "ndcg_at_5": 0.61127, "ndcg_at_10": 0.64614, "ndcg_at_20": 0.67298, "ndcg_at_50": 0.6947, "ndcg_at_100": 0.70357, "map_at_1": 0.35834, "map_at_3": 0.47597, "map_at_5": 0.51819, "map_at_10": 0.55455, "map_at_20": 0.57016, "map_at_50": 0.5782, "map_at_100": 0.58034, "recall_at_1": 0.35834, "recall_at_3": 0.54214, "recall_at_5": 0.63727, "recall_at_10": 0.74383, "recall_at_20": 0.82133, "recall_at_50": 0.88659, "recall_at_100": 0.91866, "precision_at_1": 0.57812, "precision_at_3": 0.3651, "precision_at_5": 0.28, "precision_at_10": 0.18, "precision_at_20": 0.10617, "precision_at_50": 0.05, "precision_at_100": 0.02695, "mrr_at_1": 0.5671875, "mrr_at_3": 0.6708333333333333, "mrr_at_5": 0.6866145833333334, "mrr_at_10": 0.6939360119047618, "mrr_at_20": 0.6963204666729361, "mrr_at_50": 0.6971792252566347, "mrr_at_100": 0.6973040137004454, "naucs_at_1_max": 0.22350385772789982, "naucs_at_1_std": -0.014236260379664127, "naucs_at_1_diff1": 0.415396972645202, "naucs_at_3_max": 0.05168321374193644, "naucs_at_3_std": -0.05262932689279111, "naucs_at_3_diff1": -0.046426105116023936, "naucs_at_5_max": -0.019935793098795593, "naucs_at_5_std": -0.07180552504536146, "naucs_at_5_diff1": -0.15736139387093928, "naucs_at_10_max": -0.1020058048777736, "naucs_at_10_std": -0.09370704449884888, "naucs_at_10_diff1": -0.23614803376814614, "naucs_at_20_max": -0.13810591303119216, "naucs_at_20_std": -0.0692107572861515, "naucs_at_20_diff1": -0.28493482904408224, "naucs_at_50_max": -0.17286447176352046, "naucs_at_50_std": -0.04984221651971845, "naucs_at_50_diff1": -0.317010129235068, "naucs_at_100_max": -0.18980306870058375, "naucs_at_100_std": -0.048954124365436114, "naucs_at_100_diff1": -0.3281886367637723 }, "vidore/synthetic_axa_filtered_v1.0": { "ndcg_at_1": 0.44444, "ndcg_at_3": 0.55779, "ndcg_at_5": 0.60297, "ndcg_at_10": 0.63112, "ndcg_at_20": 0.65055, "ndcg_at_50": 0.69129, "ndcg_at_100": 0.70318, "map_at_1": 0.22361, "map_at_3": 0.34019, "map_at_5": 0.42558, "map_at_10": 0.50246, "map_at_20": 0.53023, "map_at_50": 0.54858, "map_at_100": 0.55304, "recall_at_1": 0.22361, "recall_at_3": 0.43907, "recall_at_5": 0.59758, "recall_at_10": 0.75267, "recall_at_20": 0.82791, "recall_at_50": 0.94587, "recall_at_100": 0.9784, "precision_at_1": 0.44444, "precision_at_3": 0.42593, "precision_at_5": 0.4, "precision_at_10": 0.29444, "precision_at_20": 0.175, "precision_at_50": 0.08556, "precision_at_100": 0.04611, "mrr_at_1": 0.4444444444444444, "mrr_at_3": 0.6111111111111112, "mrr_at_5": 0.625, "mrr_at_10": 0.6342592592592592, "mrr_at_20": 0.6342592592592592, "mrr_at_50": 0.6367845117845118, "mrr_at_100": 0.6367845117845118, "naucs_at_1_max": 0.27180734937990775, "naucs_at_1_std": 0.1460001284215161, "naucs_at_1_diff1": 0.52680845013576, "naucs_at_3_max": -0.4472601618035251, "naucs_at_3_std": -0.35334842300017605, "naucs_at_3_diff1": -0.022038941836526545, "naucs_at_5_max": -0.5746685319786979, "naucs_at_5_std": -0.45581287349947885, "naucs_at_5_diff1": -0.02909625842935311, "naucs_at_10_max": -0.6670105211539744, "naucs_at_10_std": -0.4579454710599414, "naucs_at_10_diff1": -0.291350928264914, "naucs_at_20_max": -0.7088454526103415, "naucs_at_20_std": -0.45884776633256535, "naucs_at_20_diff1": -0.39562206735708155, "naucs_at_50_max": -0.7078100142972502, "naucs_at_50_std": -0.4315828703198929, "naucs_at_50_diff1": -0.37094602463897214, "naucs_at_100_max": -0.7059796119743438, "naucs_at_100_std": -0.400631653419931, "naucs_at_100_diff1": -0.33114628268848284 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.67241, "ndcg_at_3": 0.64169, "ndcg_at_5": 0.5978, "ndcg_at_10": 0.58577, "ndcg_at_20": 0.61496, "ndcg_at_50": 0.6772, "ndcg_at_100": 0.71432, "map_at_1": 0.08152, "map_at_3": 0.20404, "map_at_5": 0.2537, "map_at_10": 0.32889, "map_at_20": 0.39447, "map_at_50": 0.4536, "map_at_100": 0.48141, "recall_at_1": 0.08152, "recall_at_3": 0.2494, "recall_at_5": 0.32267, "recall_at_10": 0.45814, "recall_at_20": 0.61677, "recall_at_50": 0.80189, "recall_at_100": 0.92313, "precision_at_1": 0.67241, "precision_at_3": 0.59195, "precision_at_5": 0.51379, "precision_at_10": 0.42586, "precision_at_20": 0.325, "precision_at_50": 0.20138, "precision_at_100": 0.12983, "mrr_at_1": 0.6724137931034483, "mrr_at_3": 0.7729885057471265, "mrr_at_5": 0.7807471264367816, "mrr_at_10": 0.7857758620689655, "mrr_at_20": 0.7868534482758621, "mrr_at_50": 0.7876030734632684, "mrr_at_100": 0.7876030734632684, "naucs_at_1_max": 0.34454681161877004, "naucs_at_1_std": 0.5235431895381841, "naucs_at_1_diff1": -0.08387631081097016, "naucs_at_3_max": 0.08884531420395679, "naucs_at_3_std": 0.2382734233882671, "naucs_at_3_diff1": -0.1871866677817215, "naucs_at_5_max": 0.01311909348957855, "naucs_at_5_std": 0.2167381673457986, "naucs_at_5_diff1": -0.14497179289018872, "naucs_at_10_max": 0.14293369671235354, "naucs_at_10_std": 0.22265260242487017, "naucs_at_10_diff1": -0.09048933555139305, "naucs_at_20_max": 0.08115085333286959, "naucs_at_20_std": 0.20688202040577058, "naucs_at_20_diff1": -0.15069532949644948, "naucs_at_50_max": -0.010126566692404412, "naucs_at_50_std": 0.24164336672801914, "naucs_at_50_diff1": -0.10468292902065172, "naucs_at_100_max": -0.04114659355878923, "naucs_at_100_std": 0.2129831852026439, "naucs_at_100_diff1": -0.03795755522744344 }, "data_dir/eval_vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.856, "ndcg_at_3": 0.88745, "ndcg_at_5": 0.89227, "ndcg_at_10": 0.90332, "ndcg_at_20": 0.90941, "ndcg_at_50": 0.91153, "ndcg_at_100": 0.91252, "map_at_1": 0.856, "map_at_3": 0.87967, "map_at_5": 0.88227, "map_at_10": 0.88687, "map_at_20": 0.88854, "map_at_50": 0.88895, "map_at_100": 0.88904, "recall_at_1": 0.856, "recall_at_3": 0.91, "recall_at_5": 0.922, "recall_at_10": 0.956, "recall_at_20": 0.98, "recall_at_50": 0.99, "recall_at_100": 0.996, "precision_at_1": 0.856, "precision_at_3": 0.30333, "precision_at_5": 0.1844, "precision_at_10": 0.0956, "precision_at_20": 0.049, "precision_at_50": 0.0198, "precision_at_100": 0.00996, "mrr_at_1": 0.854, "mrr_at_3": 0.8773333333333334, "mrr_at_5": 0.8809333333333332, "mrr_at_10": 0.8855674603174604, "mrr_at_20": 0.8873665921453847, "mrr_at_50": 0.8876643128304966, "mrr_at_100": 0.887751797810069, "naucs_at_1_max": 0.8146679814134421, "naucs_at_1_std": 0.07396801383185674, "naucs_at_1_diff1": 0.9423560082126644, "naucs_at_3_max": 0.8066396929142029, "naucs_at_3_std": 0.07559912854030806, "naucs_at_3_diff1": 0.9181969083929873, "naucs_at_5_max": 0.8034068328185975, "naucs_at_5_std": 0.02020637314754663, "naucs_at_5_diff1": 0.9160860925566815, "naucs_at_10_max": 0.825566590272474, "naucs_at_10_std": -0.15187590187589842, "naucs_at_10_diff1": 0.9390968508615584, "naucs_at_20_max": 0.8921568627450932, "naucs_at_20_std": 0.0641923436041116, "naucs_at_20_diff1": 0.9738562091503187, "naucs_at_50_max": 0.947712418300658, "naucs_at_50_std": 0.37030812324930756, "naucs_at_50_diff1": 0.9738562091503188, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "data_dir/eval_vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.55876, "ndcg_at_3": 0.61485, "ndcg_at_5": 0.63222, "ndcg_at_10": 0.65493, "ndcg_at_20": 0.66746, "ndcg_at_50": 0.67657, "ndcg_at_100": 0.68329, "map_at_1": 0.55876, "map_at_3": 0.60126, "map_at_5": 0.6109, "map_at_10": 0.62013, "map_at_20": 0.62368, "map_at_50": 0.62509, "map_at_100": 0.62564, "recall_at_1": 0.55876, "recall_at_3": 0.6541, "recall_at_5": 0.69623, "recall_at_10": 0.76718, "recall_at_20": 0.81596, "recall_at_50": 0.86253, "recall_at_100": 0.90466, "precision_at_1": 0.55876, "precision_at_3": 0.21803, "precision_at_5": 0.13925, "precision_at_10": 0.07672, "precision_at_20": 0.0408, "precision_at_50": 0.01725, "precision_at_100": 0.00905, "mrr_at_1": 0.5543237250554324, "mrr_at_3": 0.599039172209904, "mrr_at_5": 0.609349593495935, "mrr_at_10": 0.6183401963889769, "mrr_at_20": 0.6209344044461912, "mrr_at_50": 0.6227106859869997, "mrr_at_100": 0.6232889837626537, "naucs_at_1_max": 0.29569387287927856, "naucs_at_1_std": 0.6842470452244832, "naucs_at_1_diff1": 0.9092758189268736, "naucs_at_3_max": 0.21005633503267365, "naucs_at_3_std": 0.805068083051039, "naucs_at_3_diff1": 0.8754563618133548, "naucs_at_5_max": 0.19259019824979384, "naucs_at_5_std": 0.8409490067520664, "naucs_at_5_diff1": 0.8726101181684977, "naucs_at_10_max": 0.048646855770537796, "naucs_at_10_std": 0.8833213018935924, "naucs_at_10_diff1": 0.8643899050698751, "naucs_at_20_max": -0.06225799483664109, "naucs_at_20_std": 0.9127300541361831, "naucs_at_20_diff1": 0.8597071895030789, "naucs_at_50_max": -0.24442668101014947, "naucs_at_50_std": 0.930294669379938, "naucs_at_50_diff1": 0.8714057920484658, "naucs_at_100_max": -0.3617257884810223, "naucs_at_100_std": 0.9386292283530702, "naucs_at_100_diff1": 0.8597379463433718 }, "data_dir/eval_vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.93, "ndcg_at_3": 0.95893, "ndcg_at_5": 0.95893, "ndcg_at_10": 0.95893, "ndcg_at_20": 0.96163, "ndcg_at_50": 0.96365, "ndcg_at_100": 0.96365, "map_at_1": 0.93, "map_at_3": 0.95167, "map_at_5": 0.95167, "map_at_10": 0.95167, "map_at_20": 0.9525, "map_at_50": 0.95283, "map_at_100": 0.95283, "recall_at_1": 0.93, "recall_at_3": 0.98, "recall_at_5": 0.98, "recall_at_10": 0.98, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.32667, "precision_at_5": 0.196, "precision_at_10": 0.098, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9516666666666667, "mrr_at_10": 0.9516666666666667, "mrr_at_20": 0.9525757575757575, "mrr_at_50": 0.9529205851619644, "mrr_at_100": 0.9529205851619644, "naucs_at_1_max": 0.39482459650526885, "naucs_at_1_std": -0.31419234360410914, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.6790382819794457, "naucs_at_3_std": -0.9556489262371661, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.6790382819794609, "naucs_at_5_std": -0.9556489262371534, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.6790382819794609, "naucs_at_10_std": -0.9556489262371534, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -0.1713352007469681, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "data_dir/eval_vidore/tatdqa_test": { "ndcg_at_1": 0.69927, "ndcg_at_3": 0.79372, "ndcg_at_5": 0.81105, "ndcg_at_10": 0.82459, "ndcg_at_20": 0.82878, "ndcg_at_50": 0.83303, "ndcg_at_100": 0.83442, "map_at_1": 0.69927, "map_at_3": 0.77106, "map_at_5": 0.78072, "map_at_10": 0.78653, "map_at_20": 0.7877, "map_at_50": 0.78839, "map_at_100": 0.78852, "recall_at_1": 0.69927, "recall_at_3": 0.85905, "recall_at_5": 0.90097, "recall_at_10": 0.94168, "recall_at_20": 0.95808, "recall_at_50": 0.97934, "recall_at_100": 0.98785, "precision_at_1": 0.69927, "precision_at_3": 0.28635, "precision_at_5": 0.18019, "precision_at_10": 0.09417, "precision_at_20": 0.0479, "precision_at_50": 0.01959, "precision_at_100": 0.00988, "mrr_at_1": 0.7004860267314702, "mrr_at_3": 0.7722762251923856, "mrr_at_5": 0.7813284730660186, "mrr_at_10": 0.7874199598835079, "mrr_at_20": 0.7885261582425689, "mrr_at_50": 0.7892193634018624, "mrr_at_100": 0.7893291443742743, "naucs_at_1_max": 0.2682416909968819, "naucs_at_1_std": -0.1346098281401034, "naucs_at_1_diff1": 0.8296403667835969, "naucs_at_3_max": 0.36050493227494845, "naucs_at_3_std": -0.01834048113954263, "naucs_at_3_diff1": 0.7445505884597561, "naucs_at_5_max": 0.36523716935916267, "naucs_at_5_std": 0.06489405058952001, "naucs_at_5_diff1": 0.7142046612217674, "naucs_at_10_max": 0.4293315802144752, "naucs_at_10_std": 0.2564808038730297, "naucs_at_10_diff1": 0.6873136665710184, "naucs_at_20_max": 0.4688188620078513, "naucs_at_20_std": 0.26917226500908054, "naucs_at_20_diff1": 0.6720615167289586, "naucs_at_50_max": 0.5671944664000176, "naucs_at_50_std": 0.4964253483275758, "naucs_at_50_diff1": 0.722603370462453, "naucs_at_100_max": 0.6585255212623138, "naucs_at_100_std": 0.5978336814194427, "naucs_at_100_diff1": 0.8138609714332596 }, "data_dir/eval_vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.88462, "ndcg_at_3": 0.91698, "ndcg_at_5": 0.92378, "ndcg_at_10": 0.92838, "ndcg_at_20": 0.93146, "ndcg_at_50": 0.93263, "ndcg_at_100": 0.93362, "map_at_1": 0.88462, "map_at_3": 0.90924, "map_at_5": 0.91309, "map_at_10": 0.91501, "map_at_20": 0.91585, "map_at_50": 0.91603, "map_at_100": 0.91612, "recall_at_1": 0.88462, "recall_at_3": 0.93927, "recall_at_5": 0.95547, "recall_at_10": 0.96964, "recall_at_20": 0.98178, "recall_at_50": 0.98785, "recall_at_100": 0.99393, "precision_at_1": 0.88462, "precision_at_3": 0.31309, "precision_at_5": 0.19109, "precision_at_10": 0.09696, "precision_at_20": 0.04909, "precision_at_50": 0.01976, "precision_at_100": 0.00994, "mrr_at_1": 0.8846153846153846, "mrr_at_3": 0.9092442645074225, "mrr_at_5": 0.9129892037786774, "mrr_at_10": 0.9147717049032839, "mrr_at_20": 0.9154333931688503, "mrr_at_50": 0.9156112743493511, "mrr_at_100": 0.915698522470477, "naucs_at_1_max": 0.6002033777762705, "naucs_at_1_std": -0.07102965572007143, "naucs_at_1_diff1": 0.9544715151522033, "naucs_at_3_max": 0.8025453360230541, "naucs_at_3_std": 0.19885112605813188, "naucs_at_3_diff1": 0.960819563780572, "naucs_at_5_max": 0.9121856227472513, "naucs_at_5_std": 0.3583912666662475, "naucs_at_5_diff1": 0.9584449918884831, "naucs_at_10_max": 0.9368484108193146, "naucs_at_10_std": 0.44097650713388975, "naucs_at_10_diff1": 0.9651729455827266, "naucs_at_20_max": 0.9709774546522766, "naucs_at_20_std": 0.6007213904253922, "naucs_at_20_diff1": 0.9854887273261383, "naucs_at_50_max": 0.9782330909892136, "naucs_at_50_std": 0.7577252323561762, "naucs_at_50_diff1": 0.9782330909892136, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.7075525547215259, "naucs_at_100_diff1": 0.9564661819784259 }, "data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.95, "ndcg_at_3": 0.97893, "ndcg_at_5": 0.97893, "ndcg_at_10": 0.97893, "ndcg_at_20": 0.97893, "ndcg_at_50": 0.97893, "ndcg_at_100": 0.97893, "map_at_1": 0.95, "map_at_3": 0.97167, "map_at_5": 0.97167, "map_at_10": 0.97167, "map_at_20": 0.97167, "map_at_50": 0.97167, "map_at_100": 0.97167, "recall_at_1": 0.95, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_50": 0.9783333333333333, "mrr_at_100": 0.9783333333333333, "naucs_at_1_max": 0.7605042016806716, "naucs_at_1_std": -0.35732959850606716, "naucs_at_1_diff1": 0.9738562091503253, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "data_dir/eval_vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.85714, "ndcg_at_3": 0.90523, "ndcg_at_5": 0.91122, "ndcg_at_10": 0.91807, "ndcg_at_20": 0.92163, "ndcg_at_50": 0.92385, "ndcg_at_100": 0.92385, "map_at_1": 0.85714, "map_at_3": 0.89345, "map_at_5": 0.89685, "map_at_10": 0.89962, "map_at_20": 0.90057, "map_at_50": 0.90097, "map_at_100": 0.90097, "recall_at_1": 0.85714, "recall_at_3": 0.93929, "recall_at_5": 0.95357, "recall_at_10": 0.975, "recall_at_20": 0.98929, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.85714, "precision_at_3": 0.3131, "precision_at_5": 0.19071, "precision_at_10": 0.0975, "precision_at_20": 0.04946, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8571428571428571, "mrr_at_3": 0.8928571428571429, "mrr_at_5": 0.8955357142857143, "mrr_at_10": 0.8989866780045351, "mrr_at_20": 0.8999608080411652, "mrr_at_50": 0.9003545809349381, "mrr_at_100": 0.9003545809349381, "naucs_at_1_max": 0.44880952380952394, "naucs_at_1_std": 0.06635082604470473, "naucs_at_1_diff1": 0.9110301263362479, "naucs_at_3_max": 0.7531443950129105, "naucs_at_3_std": 0.4460647003899606, "naucs_at_3_diff1": 0.9288735101883916, "naucs_at_5_max": 0.8075127486892179, "naucs_at_5_std": 0.529124470300943, "naucs_at_5_diff1": 0.917043740573152, "naucs_at_10_max": 0.7902494331065706, "naucs_at_10_std": 0.6329198346005056, "naucs_at_10_diff1": 0.9626517273576021, "naucs_at_20_max": 0.9564270152505505, "naucs_at_20_std": 0.8638344226579515, "naucs_at_20_diff1": 0.9564270152505505, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "data_dir/eval_vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.92, "ndcg_at_3": 0.96417, "ndcg_at_5": 0.96417, "ndcg_at_10": 0.9675, "ndcg_at_20": 0.9675, "ndcg_at_50": 0.9675, "ndcg_at_100": 0.9675, "map_at_1": 0.92, "map_at_3": 0.955, "map_at_5": 0.955, "map_at_10": 0.95643, "map_at_20": 0.95643, "map_at_50": 0.95643, "map_at_100": 0.95643, "recall_at_1": 0.92, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.92, "mrr_at_3": 0.955, "mrr_at_5": 0.955, "mrr_at_10": 0.9564285714285714, "mrr_at_20": 0.9564285714285714, "mrr_at_50": 0.9564285714285714, "mrr_at_100": 0.9564285714285714, "naucs_at_1_max": 0.8768674136321195, "naucs_at_1_std": 0.41940943043884304, "naucs_at_1_diff1": 0.9673202614379083, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.8692810457516356, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.8692810457516413, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "data_dir/eval_vidore/shiftproject_test": { "ndcg_at_1": 0.78, "ndcg_at_3": 0.8644, "ndcg_at_5": 0.87302, "ndcg_at_10": 0.8828, "ndcg_at_20": 0.88795, "ndcg_at_50": 0.88997, "ndcg_at_100": 0.88997, "map_at_1": 0.78, "map_at_3": 0.845, "map_at_5": 0.85, "map_at_10": 0.8541, "map_at_20": 0.85555, "map_at_50": 0.85589, "map_at_100": 0.85589, "recall_at_1": 0.78, "recall_at_3": 0.92, "recall_at_5": 0.94, "recall_at_10": 0.97, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.78, "precision_at_3": 0.30667, "precision_at_5": 0.188, "precision_at_10": 0.097, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.81, "mrr_at_3": 0.8633333333333333, "mrr_at_5": 0.8658333333333332, "mrr_at_10": 0.8699285714285715, "mrr_at_20": 0.871412087912088, "mrr_at_50": 0.8717454212454213, "mrr_at_100": 0.8717454212454213, "naucs_at_1_max": -0.24755413363008374, "naucs_at_1_std": -0.6164510594890337, "naucs_at_1_diff1": 0.8162006769601703, "naucs_at_3_max": 0.2243814192343627, "naucs_at_3_std": -0.23225957049486393, "naucs_at_3_diff1": 0.8544000933706815, "naucs_at_5_max": -0.012371615312794324, "naucs_at_5_std": -0.5504201680672273, "naucs_at_5_diff1": 0.9128540305010897, "naucs_at_10_max": -0.049486461251166146, "naucs_at_10_std": -0.27591036414565706, "naucs_at_10_diff1": 0.9128540305010848, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": 0.5541549953314738, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.99, "ndcg_at_3": 0.99631, "ndcg_at_5": 0.99631, "ndcg_at_10": 0.99631, "ndcg_at_20": 0.99631, "ndcg_at_50": 0.99631, "ndcg_at_100": 0.99631, "map_at_1": 0.99, "map_at_3": 0.995, "map_at_5": 0.995, "map_at_10": 0.995, "map_at_20": 0.995, "map_at_50": 0.995, "map_at_100": 0.995, "recall_at_1": 0.99, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.99, "mrr_at_3": 0.995, "mrr_at_5": 0.995, "mrr_at_10": 0.995, "mrr_at_20": 0.995, "mrr_at_50": 0.995, "mrr_at_100": 0.995, "naucs_at_1_max": 0.8692810457516276, "naucs_at_1_std": -0.5634920634920657, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } } }