{ "dataset_revision": "7187c21e1a532531d175eccf5a6d7c1561f6b0da", "task_name": "Vidore3FinanceEnBM25sReranking.v2", "mteb_version": "2.24.7", "scores": { "test": [ { "ndcg_at_1": 0.36893, "ndcg_at_3": 0.35438, "ndcg_at_5": 0.35379, "ndcg_at_10": 0.36579, "ndcg_at_20": 0.38361, "ndcg_at_100": 0.40589, "ndcg_at_1000": 0.40589, "map_at_1": 0.18013, "map_at_3": 0.24511, "map_at_5": 0.26852, "map_at_10": 0.2886, "map_at_20": 0.29891, "map_at_100": 0.30528, "map_at_1000": 0.30528, "recall_at_1": 0.18013, "recall_at_3": 0.27944, "recall_at_5": 0.32824, "recall_at_10": 0.39454, "recall_at_20": 0.45277, "recall_at_100": 0.52454, "recall_at_1000": 0.52454, "accuracy": 0.18013, "precision_at_1": 0.39806, "precision_at_3": 0.25351, "precision_at_5": 0.19417, "precision_at_10": 0.12492, "precision_at_20": 0.07605, "precision_at_100": 0.01877, "precision_at_1000": 0.00188, "mrr_at_1": 0.398058, "mrr_at_3": 0.470874, "mrr_at_5": 0.479612, "mrr_at_10": 0.487623, "mrr_at_20": 0.49129, "mrr_at_100": 0.492745, "mrr_at_1000": 0.492745, "nauc_ndcg_at_1_max": 0.371532, "nauc_ndcg_at_1_std": 0.237707, "nauc_ndcg_at_1_diff1": 0.329521, "nauc_ndcg_at_3_max": 0.360192, "nauc_ndcg_at_3_std": 0.252062, "nauc_ndcg_at_3_diff1": 0.259827, "nauc_ndcg_at_5_max": 0.379065, "nauc_ndcg_at_5_std": 0.269449, "nauc_ndcg_at_5_diff1": 0.260441, "nauc_ndcg_at_10_max": 0.392558, "nauc_ndcg_at_10_std": 0.282674, "nauc_ndcg_at_10_diff1": 0.254189, "nauc_ndcg_at_20_max": 0.402507, "nauc_ndcg_at_20_std": 0.294225, "nauc_ndcg_at_20_diff1": 0.251624, "nauc_ndcg_at_100_max": 0.422306, "nauc_ndcg_at_100_std": 0.316804, "nauc_ndcg_at_100_diff1": 0.246867, "nauc_ndcg_at_1000_max": 0.422306, "nauc_ndcg_at_1000_std": 0.316804, "nauc_ndcg_at_1000_diff1": 0.246867, "nauc_map_at_1_max": 0.25972, "nauc_map_at_1_std": 0.10542, "nauc_map_at_1_diff1": 0.390271, "nauc_map_at_3_max": 0.302204, "nauc_map_at_3_std": 0.167566, "nauc_map_at_3_diff1": 0.317053, "nauc_map_at_5_max": 0.3295, "nauc_map_at_5_std": 0.204625, "nauc_map_at_5_diff1": 0.288725, "nauc_map_at_10_max": 0.347739, "nauc_map_at_10_std": 0.229087, "nauc_map_at_10_diff1": 0.267579, "nauc_map_at_20_max": 0.352888, "nauc_map_at_20_std": 0.2389, "nauc_map_at_20_diff1": 0.265895, "nauc_map_at_100_max": 0.359205, "nauc_map_at_100_std": 0.246352, "nauc_map_at_100_diff1": 0.264595, "nauc_map_at_1000_max": 0.359205, "nauc_map_at_1000_std": 0.246352, "nauc_map_at_1000_diff1": 0.264595, "nauc_recall_at_1_max": 0.25972, "nauc_recall_at_1_std": 0.10542, "nauc_recall_at_1_diff1": 0.390271, "nauc_recall_at_3_max": 0.302661, "nauc_recall_at_3_std": 0.187853, "nauc_recall_at_3_diff1": 0.254969, "nauc_recall_at_5_max": 0.346931, "nauc_recall_at_5_std": 0.245809, "nauc_recall_at_5_diff1": 0.215834, "nauc_recall_at_10_max": 0.37012, "nauc_recall_at_10_std": 0.293417, "nauc_recall_at_10_diff1": 0.18111, "nauc_recall_at_20_max": 0.37958, "nauc_recall_at_20_std": 0.317694, "nauc_recall_at_20_diff1": 0.179297, "nauc_recall_at_100_max": 0.438858, "nauc_recall_at_100_std": 0.382335, "nauc_recall_at_100_diff1": 0.164611, "nauc_recall_at_1000_max": 0.438858, "nauc_recall_at_1000_std": 0.382335, "nauc_recall_at_1000_diff1": 0.164611, "nauc_precision_at_1_max": 0.40464, "nauc_precision_at_1_std": 0.282698, "nauc_precision_at_1_diff1": 0.319749, "nauc_precision_at_3_max": 0.364621, "nauc_precision_at_3_std": 0.345527, "nauc_precision_at_3_diff1": 0.072681, "nauc_precision_at_5_max": 0.36171, "nauc_precision_at_5_std": 0.371478, "nauc_precision_at_5_diff1": 0.002229, "nauc_precision_at_10_max": 0.3554, "nauc_precision_at_10_std": 0.381242, "nauc_precision_at_10_diff1": -0.057555, "nauc_precision_at_20_max": 0.316675, "nauc_precision_at_20_std": 0.367545, "nauc_precision_at_20_diff1": -0.063011, "nauc_precision_at_100_max": 0.287743, "nauc_precision_at_100_std": 0.345837, "nauc_precision_at_100_diff1": -0.079088, "nauc_precision_at_1000_max": 0.287743, "nauc_precision_at_1000_std": 0.345837, "nauc_precision_at_1000_diff1": -0.079088, "nauc_mrr_at_1_max": 0.40464, "nauc_mrr_at_1_std": 0.282698, "nauc_mrr_at_1_diff1": 0.319749, "nauc_mrr_at_3_max": 0.444968, "nauc_mrr_at_3_std": 0.338057, "nauc_mrr_at_3_diff1": 0.265832, "nauc_mrr_at_5_max": 0.449303, "nauc_mrr_at_5_std": 0.342035, "nauc_mrr_at_5_diff1": 0.263756, "nauc_mrr_at_10_max": 0.45146, "nauc_mrr_at_10_std": 0.3469, "nauc_mrr_at_10_diff1": 0.264231, "nauc_mrr_at_20_max": 0.450891, "nauc_mrr_at_20_std": 0.346308, "nauc_mrr_at_20_diff1": 0.26506, "nauc_mrr_at_100_max": 0.451402, "nauc_mrr_at_100_std": 0.346921, "nauc_mrr_at_100_diff1": 0.265357, "nauc_mrr_at_1000_max": 0.451402, "nauc_mrr_at_1000_std": 0.346921, "nauc_mrr_at_1000_diff1": 0.265357, "hit_rate_at_1": 0.39806, "hit_rate_at_3": 0.55663, "hit_rate_at_5": 0.59223, "hit_rate_at_10": 0.65049, "hit_rate_at_20": 0.7055, "hit_rate_at_100": 0.75081, "hit_rate_at_1000": 0.75081, "main_score": 0.36579, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.38997, "ndcg_at_3": 0.37407, "ndcg_at_5": 0.3751, "ndcg_at_10": 0.38433, "ndcg_at_20": 0.39964, "ndcg_at_100": 0.42524, "ndcg_at_1000": 0.42524, "map_at_1": 0.18569, "map_at_3": 0.25639, "map_at_5": 0.28139, "map_at_10": 0.29907, "map_at_20": 0.30855, "map_at_100": 0.31648, "map_at_1000": 0.31648, "recall_at_1": 0.18569, "recall_at_3": 0.29433, "recall_at_5": 0.35127, "recall_at_10": 0.4122, "recall_at_20": 0.46613, "recall_at_100": 0.55092, "recall_at_1000": 0.55092, "accuracy": 0.18569, "precision_at_1": 0.41424, "precision_at_3": 0.26861, "precision_at_5": 0.20453, "precision_at_10": 0.13074, "precision_at_20": 0.0788, "precision_at_100": 0.02, "precision_at_1000": 0.002, "mrr_at_1": 0.414239, "mrr_at_3": 0.478964, "mrr_at_5": 0.494498, "mrr_at_10": 0.503573, "mrr_at_20": 0.506486, "mrr_at_100": 0.507689, "mrr_at_1000": 0.507689, "nauc_ndcg_at_1_max": 0.486172, "nauc_ndcg_at_1_std": 0.275648, "nauc_ndcg_at_1_diff1": 0.353836, "nauc_ndcg_at_3_max": 0.45373, "nauc_ndcg_at_3_std": 0.305887, "nauc_ndcg_at_3_diff1": 0.258071, "nauc_ndcg_at_5_max": 0.436606, "nauc_ndcg_at_5_std": 0.29535, "nauc_ndcg_at_5_diff1": 0.269244, "nauc_ndcg_at_10_max": 0.448333, "nauc_ndcg_at_10_std": 0.307382, "nauc_ndcg_at_10_diff1": 0.273798, "nauc_ndcg_at_20_max": 0.468972, "nauc_ndcg_at_20_std": 0.327266, "nauc_ndcg_at_20_diff1": 0.272061, "nauc_ndcg_at_100_max": 0.481679, "nauc_ndcg_at_100_std": 0.345695, "nauc_ndcg_at_100_diff1": 0.264117, "nauc_ndcg_at_1000_max": 0.481679, "nauc_ndcg_at_1000_std": 0.345695, "nauc_ndcg_at_1000_diff1": 0.264117, "nauc_map_at_1_max": 0.35223, "nauc_map_at_1_std": 0.150553, "nauc_map_at_1_diff1": 0.408556, "nauc_map_at_3_max": 0.389566, "nauc_map_at_3_std": 0.222794, "nauc_map_at_3_diff1": 0.312956, "nauc_map_at_5_max": 0.396363, "nauc_map_at_5_std": 0.243641, "nauc_map_at_5_diff1": 0.28829, "nauc_map_at_10_max": 0.415979, "nauc_map_at_10_std": 0.27033, "nauc_map_at_10_diff1": 0.274453, "nauc_map_at_20_max": 0.425816, "nauc_map_at_20_std": 0.279707, "nauc_map_at_20_diff1": 0.272613, "nauc_map_at_100_max": 0.431014, "nauc_map_at_100_std": 0.286512, "nauc_map_at_100_diff1": 0.268638, "nauc_map_at_1000_max": 0.431014, "nauc_map_at_1000_std": 0.286512, "nauc_map_at_1000_diff1": 0.268638, "nauc_recall_at_1_max": 0.35223, "nauc_recall_at_1_std": 0.150553, "nauc_recall_at_1_diff1": 0.408556, "nauc_recall_at_3_max": 0.356767, "nauc_recall_at_3_std": 0.230327, "nauc_recall_at_3_diff1": 0.258067, "nauc_recall_at_5_max": 0.328175, "nauc_recall_at_5_std": 0.229075, "nauc_recall_at_5_diff1": 0.235438, "nauc_recall_at_10_max": 0.351347, "nauc_recall_at_10_std": 0.261687, "nauc_recall_at_10_diff1": 0.228621, "nauc_recall_at_20_max": 0.398566, "nauc_recall_at_20_std": 0.309733, "nauc_recall_at_20_diff1": 0.233547, "nauc_recall_at_100_max": 0.424247, "nauc_recall_at_100_std": 0.355257, "nauc_recall_at_100_diff1": 0.206547, "nauc_recall_at_1000_max": 0.424247, "nauc_recall_at_1000_std": 0.355257, "nauc_recall_at_1000_diff1": 0.206547, "nauc_precision_at_1_max": 0.497746, "nauc_precision_at_1_std": 0.299021, "nauc_precision_at_1_diff1": 0.352436, "nauc_precision_at_3_max": 0.406303, "nauc_precision_at_3_std": 0.375313, "nauc_precision_at_3_diff1": 0.031666, "nauc_precision_at_5_max": 0.351093, "nauc_precision_at_5_std": 0.360308, "nauc_precision_at_5_diff1": -0.017238, "nauc_precision_at_10_max": 0.349319, "nauc_precision_at_10_std": 0.369912, "nauc_precision_at_10_diff1": -0.042711, "nauc_precision_at_20_max": 0.328372, "nauc_precision_at_20_std": 0.343653, "nauc_precision_at_20_diff1": -0.052309, "nauc_precision_at_100_max": 0.267877, "nauc_precision_at_100_std": 0.298602, "nauc_precision_at_100_diff1": -0.083797, "nauc_precision_at_1000_max": 0.267877, "nauc_precision_at_1000_std": 0.298602, "nauc_precision_at_1000_diff1": -0.083797, "nauc_mrr_at_1_max": 0.497746, "nauc_mrr_at_1_std": 0.299021, "nauc_mrr_at_1_diff1": 0.352436, "nauc_mrr_at_3_max": 0.510592, "nauc_mrr_at_3_std": 0.356188, "nauc_mrr_at_3_diff1": 0.297506, "nauc_mrr_at_5_max": 0.495712, "nauc_mrr_at_5_std": 0.343134, "nauc_mrr_at_5_diff1": 0.306633, "nauc_mrr_at_10_max": 0.494481, "nauc_mrr_at_10_std": 0.340424, "nauc_mrr_at_10_diff1": 0.309732, "nauc_mrr_at_20_max": 0.498564, "nauc_mrr_at_20_std": 0.344681, "nauc_mrr_at_20_diff1": 0.309979, "nauc_mrr_at_100_max": 0.498843, "nauc_mrr_at_100_std": 0.344568, "nauc_mrr_at_100_diff1": 0.310159, "nauc_mrr_at_1000_max": 0.498843, "nauc_mrr_at_1000_std": 0.344568, "nauc_mrr_at_1000_diff1": 0.310159, "hit_rate_at_1": 0.41424, "hit_rate_at_3": 0.55987, "hit_rate_at_5": 0.62783, "hit_rate_at_10": 0.69903, "hit_rate_at_20": 0.7411, "hit_rate_at_100": 0.77994, "hit_rate_at_1000": 0.77994, "main_score": 0.38433, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.48706, "ndcg_at_3": 0.48326, "ndcg_at_5": 0.48708, "ndcg_at_10": 0.50909, "ndcg_at_20": 0.5405, "ndcg_at_100": 0.58463, "ndcg_at_1000": 0.58463, "map_at_1": 0.23407, "map_at_3": 0.33738, "map_at_5": 0.36934, "map_at_10": 0.40325, "map_at_20": 0.42252, "map_at_100": 0.43718, "map_at_1000": 0.43718, "recall_at_1": 0.23407, "recall_at_3": 0.38874, "recall_at_5": 0.45429, "recall_at_10": 0.55689, "recall_at_20": 0.65704, "recall_at_100": 0.8041, "recall_at_1000": 0.8041, "accuracy": 0.23407, "precision_at_1": 0.52751, "precision_at_3": 0.35059, "precision_at_5": 0.27573, "precision_at_10": 0.18414, "precision_at_20": 0.11667, "precision_at_100": 0.03049, "precision_at_1000": 0.00305, "mrr_at_1": 0.527508, "mrr_at_3": 0.614887, "mrr_at_5": 0.630906, "mrr_at_10": 0.638484, "mrr_at_20": 0.643049, "mrr_at_100": 0.645082, "mrr_at_1000": 0.645082, "nauc_ndcg_at_1_max": 0.3149, "nauc_ndcg_at_1_std": -0.032403, "nauc_ndcg_at_1_diff1": 0.195006, "nauc_ndcg_at_3_max": 0.275931, "nauc_ndcg_at_3_std": -0.056448, "nauc_ndcg_at_3_diff1": 0.153892, "nauc_ndcg_at_5_max": 0.280194, "nauc_ndcg_at_5_std": -0.05134, "nauc_ndcg_at_5_diff1": 0.145793, "nauc_ndcg_at_10_max": 0.308007, "nauc_ndcg_at_10_std": -0.015788, "nauc_ndcg_at_10_diff1": 0.139962, "nauc_ndcg_at_20_max": 0.33301, "nauc_ndcg_at_20_std": 0.01766, "nauc_ndcg_at_20_diff1": 0.131489, "nauc_ndcg_at_100_max": 0.344253, "nauc_ndcg_at_100_std": 0.039786, "nauc_ndcg_at_100_diff1": 0.106165, "nauc_ndcg_at_1000_max": 0.344253, "nauc_ndcg_at_1000_std": 0.039786, "nauc_ndcg_at_1000_diff1": 0.106165, "nauc_map_at_1_max": 0.197473, "nauc_map_at_1_std": -0.11868, "nauc_map_at_1_diff1": 0.304579, "nauc_map_at_3_max": 0.21747, "nauc_map_at_3_std": -0.109203, "nauc_map_at_3_diff1": 0.221263, "nauc_map_at_5_max": 0.24034, "nauc_map_at_5_std": -0.08673, "nauc_map_at_5_diff1": 0.187843, "nauc_map_at_10_max": 0.271928, "nauc_map_at_10_std": -0.048974, "nauc_map_at_10_diff1": 0.164085, "nauc_map_at_20_max": 0.28329, "nauc_map_at_20_std": -0.029006, "nauc_map_at_20_diff1": 0.153742, "nauc_map_at_100_max": 0.28802, "nauc_map_at_100_std": -0.020553, "nauc_map_at_100_diff1": 0.145025, "nauc_map_at_1000_max": 0.28802, "nauc_map_at_1000_std": -0.020553, "nauc_map_at_1000_diff1": 0.145025, "nauc_recall_at_1_max": 0.197473, "nauc_recall_at_1_std": -0.11868, "nauc_recall_at_1_diff1": 0.304579, "nauc_recall_at_3_max": 0.199261, "nauc_recall_at_3_std": -0.101843, "nauc_recall_at_3_diff1": 0.185456, "nauc_recall_at_5_max": 0.222869, "nauc_recall_at_5_std": -0.063552, "nauc_recall_at_5_diff1": 0.126736, "nauc_recall_at_10_max": 0.265627, "nauc_recall_at_10_std": 0.011839, "nauc_recall_at_10_diff1": 0.082417, "nauc_recall_at_20_max": 0.300424, "nauc_recall_at_20_std": 0.102622, "nauc_recall_at_20_diff1": 0.048029, "nauc_recall_at_100_max": 0.373834, "nauc_recall_at_100_std": 0.275996, "nauc_recall_at_100_diff1": -0.122141, "nauc_recall_at_1000_max": 0.373834, "nauc_recall_at_1000_std": 0.275996, "nauc_recall_at_1000_diff1": -0.122141, "nauc_precision_at_1_max": 0.327793, "nauc_precision_at_1_std": -0.007647, "nauc_precision_at_1_diff1": 0.195509, "nauc_precision_at_3_max": 0.237556, "nauc_precision_at_3_std": 0.050546, "nauc_precision_at_3_diff1": -0.07031, "nauc_precision_at_5_max": 0.221233, "nauc_precision_at_5_std": 0.112439, "nauc_precision_at_5_diff1": -0.149995, "nauc_precision_at_10_max": 0.201211, "nauc_precision_at_10_std": 0.17168, "nauc_precision_at_10_diff1": -0.174459, "nauc_precision_at_20_max": 0.153725, "nauc_precision_at_20_std": 0.200886, "nauc_precision_at_20_diff1": -0.184585, "nauc_precision_at_100_max": 0.069144, "nauc_precision_at_100_std": 0.182356, "nauc_precision_at_100_diff1": -0.208375, "nauc_precision_at_1000_max": 0.069144, "nauc_precision_at_1000_std": 0.182356, "nauc_precision_at_1000_diff1": -0.208375, "nauc_mrr_at_1_max": 0.327793, "nauc_mrr_at_1_std": -0.007647, "nauc_mrr_at_1_diff1": 0.195509, "nauc_mrr_at_3_max": 0.339859, "nauc_mrr_at_3_std": -0.011503, "nauc_mrr_at_3_diff1": 0.171066, "nauc_mrr_at_5_max": 0.342601, "nauc_mrr_at_5_std": -0.004638, "nauc_mrr_at_5_diff1": 0.160777, "nauc_mrr_at_10_max": 0.340949, "nauc_mrr_at_10_std": -0.004107, "nauc_mrr_at_10_diff1": 0.167613, "nauc_mrr_at_20_max": 0.343979, "nauc_mrr_at_20_std": -0.000757, "nauc_mrr_at_20_diff1": 0.168159, "nauc_mrr_at_100_max": 0.343311, "nauc_mrr_at_100_std": -0.001808, "nauc_mrr_at_100_diff1": 0.168006, "nauc_mrr_at_1000_max": 0.343311, "nauc_mrr_at_1000_std": -0.001808, "nauc_mrr_at_1000_diff1": 0.168006, "hit_rate_at_1": 0.52751, "hit_rate_at_3": 0.72168, "hit_rate_at_5": 0.79288, "hit_rate_at_10": 0.85113, "hit_rate_at_20": 0.91262, "hit_rate_at_100": 0.97411, "hit_rate_at_1000": 0.97411, "main_score": 0.50909, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.35761, "ndcg_at_3": 0.32907, "ndcg_at_5": 0.3276, "ndcg_at_10": 0.336, "ndcg_at_20": 0.35093, "ndcg_at_100": 0.37739, "ndcg_at_1000": 0.37739, "map_at_1": 0.16306, "map_at_3": 0.22503, "map_at_5": 0.24669, "map_at_10": 0.26244, "map_at_20": 0.27039, "map_at_100": 0.27794, "map_at_1000": 0.27794, "recall_at_1": 0.16306, "recall_at_3": 0.25625, "recall_at_5": 0.30511, "recall_at_10": 0.35549, "recall_at_20": 0.41162, "recall_at_100": 0.50049, "recall_at_1000": 0.50049, "accuracy": 0.16306, "precision_at_1": 0.38511, "precision_at_3": 0.23625, "precision_at_5": 0.17994, "precision_at_10": 0.11553, "precision_at_20": 0.06958, "precision_at_100": 0.01812, "precision_at_1000": 0.00181, "mrr_at_1": 0.385113, "mrr_at_3": 0.43959, "mrr_at_5": 0.452859, "mrr_at_10": 0.458708, "mrr_at_20": 0.462931, "mrr_at_100": 0.464278, "mrr_at_1000": 0.464278, "nauc_ndcg_at_1_max": 0.394825, "nauc_ndcg_at_1_std": 0.237607, "nauc_ndcg_at_1_diff1": 0.258, "nauc_ndcg_at_3_max": 0.412127, "nauc_ndcg_at_3_std": 0.270583, "nauc_ndcg_at_3_diff1": 0.239914, "nauc_ndcg_at_5_max": 0.423143, "nauc_ndcg_at_5_std": 0.284871, "nauc_ndcg_at_5_diff1": 0.234978, "nauc_ndcg_at_10_max": 0.426548, "nauc_ndcg_at_10_std": 0.293236, "nauc_ndcg_at_10_diff1": 0.228285, "nauc_ndcg_at_20_max": 0.445331, "nauc_ndcg_at_20_std": 0.313841, "nauc_ndcg_at_20_diff1": 0.225121, "nauc_ndcg_at_100_max": 0.451673, "nauc_ndcg_at_100_std": 0.330626, "nauc_ndcg_at_100_diff1": 0.226749, "nauc_ndcg_at_1000_max": 0.451673, "nauc_ndcg_at_1000_std": 0.330626, "nauc_ndcg_at_1000_diff1": 0.226749, "nauc_map_at_1_max": 0.335761, "nauc_map_at_1_std": 0.172428, "nauc_map_at_1_diff1": 0.353011, "nauc_map_at_3_max": 0.35199, "nauc_map_at_3_std": 0.205124, "nauc_map_at_3_diff1": 0.278473, "nauc_map_at_5_max": 0.363611, "nauc_map_at_5_std": 0.226994, "nauc_map_at_5_diff1": 0.250059, "nauc_map_at_10_max": 0.374517, "nauc_map_at_10_std": 0.241629, "nauc_map_at_10_diff1": 0.237999, "nauc_map_at_20_max": 0.38284, "nauc_map_at_20_std": 0.251846, "nauc_map_at_20_diff1": 0.23262, "nauc_map_at_100_max": 0.386065, "nauc_map_at_100_std": 0.257379, "nauc_map_at_100_diff1": 0.23366, "nauc_map_at_1000_max": 0.386065, "nauc_map_at_1000_std": 0.257379, "nauc_map_at_1000_diff1": 0.23366, "nauc_recall_at_1_max": 0.335761, "nauc_recall_at_1_std": 0.172428, "nauc_recall_at_1_diff1": 0.353011, "nauc_recall_at_3_max": 0.363505, "nauc_recall_at_3_std": 0.236392, "nauc_recall_at_3_diff1": 0.235683, "nauc_recall_at_5_max": 0.366543, "nauc_recall_at_5_std": 0.264969, "nauc_recall_at_5_diff1": 0.179945, "nauc_recall_at_10_max": 0.373073, "nauc_recall_at_10_std": 0.28603, "nauc_recall_at_10_diff1": 0.15498, "nauc_recall_at_20_max": 0.404609, "nauc_recall_at_20_std": 0.328686, "nauc_recall_at_20_diff1": 0.137241, "nauc_recall_at_100_max": 0.412831, "nauc_recall_at_100_std": 0.368761, "nauc_recall_at_100_diff1": 0.126176, "nauc_recall_at_1000_max": 0.412831, "nauc_recall_at_1000_std": 0.368761, "nauc_recall_at_1000_diff1": 0.126176, "nauc_precision_at_1_max": 0.417405, "nauc_precision_at_1_std": 0.261289, "nauc_precision_at_1_diff1": 0.245123, "nauc_precision_at_3_max": 0.369411, "nauc_precision_at_3_std": 0.301629, "nauc_precision_at_3_diff1": 0.057868, "nauc_precision_at_5_max": 0.361753, "nauc_precision_at_5_std": 0.323049, "nauc_precision_at_5_diff1": -0.00073, "nauc_precision_at_10_max": 0.32712, "nauc_precision_at_10_std": 0.309246, "nauc_precision_at_10_diff1": -0.024582, "nauc_precision_at_20_max": 0.337739, "nauc_precision_at_20_std": 0.324891, "nauc_precision_at_20_diff1": -0.024774, "nauc_precision_at_100_max": 0.253509, "nauc_precision_at_100_std": 0.268331, "nauc_precision_at_100_diff1": -0.010578, "nauc_precision_at_1000_max": 0.253509, "nauc_precision_at_1000_std": 0.268331, "nauc_precision_at_1000_diff1": -0.010578, "nauc_mrr_at_1_max": 0.417405, "nauc_mrr_at_1_std": 0.261289, "nauc_mrr_at_1_diff1": 0.245123, "nauc_mrr_at_3_max": 0.466369, "nauc_mrr_at_3_std": 0.326426, "nauc_mrr_at_3_diff1": 0.223755, "nauc_mrr_at_5_max": 0.472538, "nauc_mrr_at_5_std": 0.336437, "nauc_mrr_at_5_diff1": 0.2175, "nauc_mrr_at_10_max": 0.4673, "nauc_mrr_at_10_std": 0.332643, "nauc_mrr_at_10_diff1": 0.216995, "nauc_mrr_at_20_max": 0.468932, "nauc_mrr_at_20_std": 0.334549, "nauc_mrr_at_20_diff1": 0.220094, "nauc_mrr_at_100_max": 0.469201, "nauc_mrr_at_100_std": 0.335444, "nauc_mrr_at_100_diff1": 0.219438, "nauc_mrr_at_1000_max": 0.469201, "nauc_mrr_at_1000_std": 0.335444, "nauc_mrr_at_1000_diff1": 0.219438, "hit_rate_at_1": 0.38511, "hit_rate_at_3": 0.50809, "hit_rate_at_5": 0.56634, "hit_rate_at_10": 0.60518, "hit_rate_at_20": 0.6699, "hit_rate_at_100": 0.71197, "hit_rate_at_1000": 0.71197, "main_score": 0.336, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.37055, "ndcg_at_3": 0.35661, "ndcg_at_5": 0.354, "ndcg_at_10": 0.36062, "ndcg_at_20": 0.37867, "ndcg_at_100": 0.40581, "ndcg_at_1000": 0.40581, "map_at_1": 0.16534, "map_at_3": 0.24343, "map_at_5": 0.26541, "map_at_10": 0.28002, "map_at_20": 0.29004, "map_at_100": 0.29794, "map_at_1000": 0.29794, "recall_at_1": 0.16534, "recall_at_3": 0.29168, "recall_at_5": 0.33711, "recall_at_10": 0.38791, "recall_at_20": 0.44562, "recall_at_100": 0.53718, "recall_at_1000": 0.53718, "accuracy": 0.16534, "precision_at_1": 0.40129, "precision_at_3": 0.2589, "precision_at_5": 0.19676, "precision_at_10": 0.12071, "precision_at_20": 0.07476, "precision_at_100": 0.01909, "precision_at_1000": 0.00191, "mrr_at_1": 0.401294, "mrr_at_3": 0.472492, "mrr_at_5": 0.483333, "mrr_at_10": 0.490537, "mrr_at_20": 0.494751, "mrr_at_100": 0.49628, "mrr_at_1000": 0.49628, "nauc_ndcg_at_1_max": 0.375177, "nauc_ndcg_at_1_std": 0.149948, "nauc_ndcg_at_1_diff1": 0.414823, "nauc_ndcg_at_3_max": 0.360975, "nauc_ndcg_at_3_std": 0.191189, "nauc_ndcg_at_3_diff1": 0.357197, "nauc_ndcg_at_5_max": 0.374887, "nauc_ndcg_at_5_std": 0.215367, "nauc_ndcg_at_5_diff1": 0.33541, "nauc_ndcg_at_10_max": 0.383079, "nauc_ndcg_at_10_std": 0.229229, "nauc_ndcg_at_10_diff1": 0.331172, "nauc_ndcg_at_20_max": 0.398079, "nauc_ndcg_at_20_std": 0.250131, "nauc_ndcg_at_20_diff1": 0.323126, "nauc_ndcg_at_100_max": 0.409524, "nauc_ndcg_at_100_std": 0.269562, "nauc_ndcg_at_100_diff1": 0.305194, "nauc_ndcg_at_1000_max": 0.409524, "nauc_ndcg_at_1000_std": 0.269562, "nauc_ndcg_at_1000_diff1": 0.305194, "nauc_map_at_1_max": 0.267674, "nauc_map_at_1_std": 0.058292, "nauc_map_at_1_diff1": 0.453252, "nauc_map_at_3_max": 0.292186, "nauc_map_at_3_std": 0.128992, "nauc_map_at_3_diff1": 0.365089, "nauc_map_at_5_max": 0.312212, "nauc_map_at_5_std": 0.163495, "nauc_map_at_5_diff1": 0.326559, "nauc_map_at_10_max": 0.32182, "nauc_map_at_10_std": 0.181319, "nauc_map_at_10_diff1": 0.316287, "nauc_map_at_20_max": 0.332552, "nauc_map_at_20_std": 0.195667, "nauc_map_at_20_diff1": 0.313491, "nauc_map_at_100_max": 0.338647, "nauc_map_at_100_std": 0.204662, "nauc_map_at_100_diff1": 0.307175, "nauc_map_at_1000_max": 0.338647, "nauc_map_at_1000_std": 0.204662, "nauc_map_at_1000_diff1": 0.307175, "nauc_recall_at_1_max": 0.267674, "nauc_recall_at_1_std": 0.058292, "nauc_recall_at_1_diff1": 0.453252, "nauc_recall_at_3_max": 0.287141, "nauc_recall_at_3_std": 0.162755, "nauc_recall_at_3_diff1": 0.297149, "nauc_recall_at_5_max": 0.316835, "nauc_recall_at_5_std": 0.224089, "nauc_recall_at_5_diff1": 0.237093, "nauc_recall_at_10_max": 0.319124, "nauc_recall_at_10_std": 0.240373, "nauc_recall_at_10_diff1": 0.227692, "nauc_recall_at_20_max": 0.359333, "nauc_recall_at_20_std": 0.290597, "nauc_recall_at_20_diff1": 0.221048, "nauc_recall_at_100_max": 0.373504, "nauc_recall_at_100_std": 0.3335, "nauc_recall_at_100_diff1": 0.152312, "nauc_recall_at_1000_max": 0.373504, "nauc_recall_at_1000_std": 0.3335, "nauc_recall_at_1000_diff1": 0.152312, "nauc_precision_at_1_max": 0.404003, "nauc_precision_at_1_std": 0.175487, "nauc_precision_at_1_diff1": 0.412703, "nauc_precision_at_3_max": 0.35588, "nauc_precision_at_3_std": 0.284581, "nauc_precision_at_3_diff1": 0.16282, "nauc_precision_at_5_max": 0.351291, "nauc_precision_at_5_std": 0.321103, "nauc_precision_at_5_diff1": 0.071089, "nauc_precision_at_10_max": 0.340181, "nauc_precision_at_10_std": 0.337581, "nauc_precision_at_10_diff1": 0.044337, "nauc_precision_at_20_max": 0.327492, "nauc_precision_at_20_std": 0.344122, "nauc_precision_at_20_diff1": 0.022604, "nauc_precision_at_100_max": 0.28926, "nauc_precision_at_100_std": 0.329382, "nauc_precision_at_100_diff1": -0.047901, "nauc_precision_at_1000_max": 0.28926, "nauc_precision_at_1000_std": 0.329382, "nauc_precision_at_1000_diff1": -0.047901, "nauc_mrr_at_1_max": 0.404003, "nauc_mrr_at_1_std": 0.175487, "nauc_mrr_at_1_diff1": 0.412703, "nauc_mrr_at_3_max": 0.450868, "nauc_mrr_at_3_std": 0.239981, "nauc_mrr_at_3_diff1": 0.393991, "nauc_mrr_at_5_max": 0.456798, "nauc_mrr_at_5_std": 0.256062, "nauc_mrr_at_5_diff1": 0.384169, "nauc_mrr_at_10_max": 0.452283, "nauc_mrr_at_10_std": 0.251579, "nauc_mrr_at_10_diff1": 0.389556, "nauc_mrr_at_20_max": 0.449999, "nauc_mrr_at_20_std": 0.249747, "nauc_mrr_at_20_diff1": 0.387042, "nauc_mrr_at_100_max": 0.44932, "nauc_mrr_at_100_std": 0.249392, "nauc_mrr_at_100_diff1": 0.386609, "nauc_mrr_at_1000_max": 0.44932, "nauc_mrr_at_1000_std": 0.249392, "nauc_mrr_at_1000_diff1": 0.386609, "hit_rate_at_1": 0.40129, "hit_rate_at_3": 0.56311, "hit_rate_at_5": 0.61165, "hit_rate_at_10": 0.66343, "hit_rate_at_20": 0.72492, "hit_rate_at_100": 0.76699, "hit_rate_at_1000": 0.76699, "main_score": 0.36062, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.34628, "ndcg_at_3": 0.32994, "ndcg_at_5": 0.33061, "ndcg_at_10": 0.34341, "ndcg_at_20": 0.3567, "ndcg_at_100": 0.38135, "ndcg_at_1000": 0.38135, "map_at_1": 0.16602, "map_at_3": 0.23028, "map_at_5": 0.25001, "map_at_10": 0.26767, "map_at_20": 0.27549, "map_at_100": 0.28224, "map_at_1000": 0.28224, "recall_at_1": 0.16602, "recall_at_3": 0.26361, "recall_at_5": 0.30765, "recall_at_10": 0.36979, "recall_at_20": 0.41678, "recall_at_100": 0.50066, "recall_at_1000": 0.50066, "accuracy": 0.16602, "precision_at_1": 0.37217, "precision_at_3": 0.23517, "precision_at_5": 0.17994, "precision_at_10": 0.1165, "precision_at_20": 0.06861, "precision_at_100": 0.0177, "precision_at_1000": 0.00177, "mrr_at_1": 0.372168, "mrr_at_3": 0.430421, "mrr_at_5": 0.444337, "mrr_at_10": 0.45416, "mrr_at_20": 0.457732, "mrr_at_100": 0.459483, "mrr_at_1000": 0.459483, "nauc_ndcg_at_1_max": 0.408929, "nauc_ndcg_at_1_std": 0.175411, "nauc_ndcg_at_1_diff1": 0.333643, "nauc_ndcg_at_3_max": 0.406908, "nauc_ndcg_at_3_std": 0.217519, "nauc_ndcg_at_3_diff1": 0.252203, "nauc_ndcg_at_5_max": 0.419574, "nauc_ndcg_at_5_std": 0.227552, "nauc_ndcg_at_5_diff1": 0.253452, "nauc_ndcg_at_10_max": 0.422783, "nauc_ndcg_at_10_std": 0.243323, "nauc_ndcg_at_10_diff1": 0.239425, "nauc_ndcg_at_20_max": 0.438267, "nauc_ndcg_at_20_std": 0.260233, "nauc_ndcg_at_20_diff1": 0.249817, "nauc_ndcg_at_100_max": 0.448213, "nauc_ndcg_at_100_std": 0.276808, "nauc_ndcg_at_100_diff1": 0.240832, "nauc_ndcg_at_1000_max": 0.448213, "nauc_ndcg_at_1000_std": 0.276808, "nauc_ndcg_at_1000_diff1": 0.240832, "nauc_map_at_1_max": 0.278052, "nauc_map_at_1_std": 0.059732, "nauc_map_at_1_diff1": 0.374648, "nauc_map_at_3_max": 0.339782, "nauc_map_at_3_std": 0.142158, "nauc_map_at_3_diff1": 0.309971, "nauc_map_at_5_max": 0.369002, "nauc_map_at_5_std": 0.169164, "nauc_map_at_5_diff1": 0.296975, "nauc_map_at_10_max": 0.382338, "nauc_map_at_10_std": 0.19715, "nauc_map_at_10_diff1": 0.272136, "nauc_map_at_20_max": 0.389852, "nauc_map_at_20_std": 0.205603, "nauc_map_at_20_diff1": 0.273585, "nauc_map_at_100_max": 0.393417, "nauc_map_at_100_std": 0.210932, "nauc_map_at_100_diff1": 0.271198, "nauc_map_at_1000_max": 0.393417, "nauc_map_at_1000_std": 0.210932, "nauc_map_at_1000_diff1": 0.271198, "nauc_recall_at_1_max": 0.278052, "nauc_recall_at_1_std": 0.059732, "nauc_recall_at_1_diff1": 0.374648, "nauc_recall_at_3_max": 0.355828, "nauc_recall_at_3_std": 0.192418, "nauc_recall_at_3_diff1": 0.261649, "nauc_recall_at_5_max": 0.384706, "nauc_recall_at_5_std": 0.228131, "nauc_recall_at_5_diff1": 0.236962, "nauc_recall_at_10_max": 0.374013, "nauc_recall_at_10_std": 0.255926, "nauc_recall_at_10_diff1": 0.179715, "nauc_recall_at_20_max": 0.399984, "nauc_recall_at_20_std": 0.290347, "nauc_recall_at_20_diff1": 0.201626, "nauc_recall_at_100_max": 0.41378, "nauc_recall_at_100_std": 0.33042, "nauc_recall_at_100_diff1": 0.15518, "nauc_recall_at_1000_max": 0.41378, "nauc_recall_at_1000_std": 0.33042, "nauc_recall_at_1000_diff1": 0.15518, "nauc_precision_at_1_max": 0.442373, "nauc_precision_at_1_std": 0.211983, "nauc_precision_at_1_diff1": 0.336396, "nauc_precision_at_3_max": 0.450902, "nauc_precision_at_3_std": 0.335232, "nauc_precision_at_3_diff1": 0.0863, "nauc_precision_at_5_max": 0.441652, "nauc_precision_at_5_std": 0.347159, "nauc_precision_at_5_diff1": 0.031846, "nauc_precision_at_10_max": 0.401216, "nauc_precision_at_10_std": 0.368268, "nauc_precision_at_10_diff1": -0.03817, "nauc_precision_at_20_max": 0.383558, "nauc_precision_at_20_std": 0.367565, "nauc_precision_at_20_diff1": -0.02042, "nauc_precision_at_100_max": 0.306597, "nauc_precision_at_100_std": 0.317604, "nauc_precision_at_100_diff1": -0.030196, "nauc_precision_at_1000_max": 0.306597, "nauc_precision_at_1000_std": 0.317604, "nauc_precision_at_1000_diff1": -0.030196, "nauc_mrr_at_1_max": 0.442373, "nauc_mrr_at_1_std": 0.211983, "nauc_mrr_at_1_diff1": 0.336396, "nauc_mrr_at_3_max": 0.488768, "nauc_mrr_at_3_std": 0.279937, "nauc_mrr_at_3_diff1": 0.28217, "nauc_mrr_at_5_max": 0.483034, "nauc_mrr_at_5_std": 0.280097, "nauc_mrr_at_5_diff1": 0.277406, "nauc_mrr_at_10_max": 0.47566, "nauc_mrr_at_10_std": 0.275729, "nauc_mrr_at_10_diff1": 0.273297, "nauc_mrr_at_20_max": 0.477247, "nauc_mrr_at_20_std": 0.277706, "nauc_mrr_at_20_diff1": 0.276441, "nauc_mrr_at_100_max": 0.476719, "nauc_mrr_at_100_std": 0.27751, "nauc_mrr_at_100_diff1": 0.274739, "nauc_mrr_at_1000_max": 0.476719, "nauc_mrr_at_1000_std": 0.27751, "nauc_mrr_at_1000_diff1": 0.274739, "hit_rate_at_1": 0.37217, "hit_rate_at_3": 0.50485, "hit_rate_at_5": 0.56634, "hit_rate_at_10": 0.6343, "hit_rate_at_20": 0.68285, "hit_rate_at_100": 0.73139, "hit_rate_at_1000": 0.73139, "main_score": 0.34341, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.24.7" } ] }, "evaluation_time": 1245.7260746009997, "kg_co2_emissions": null, "date": 1791746106.79908, "evaluation_phases": [ { "name": "Data loading", "start": 5.468005838338286e-06, "end": 71.51055904900568, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 71.51057889900403, "end": 71.51058057200134, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 71.51522526600456, "end": 273.7017114150003, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 273.7017731070009, "end": 273.74585995200323, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 273.75770061000367, "end": 467.96134178900684, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 467.9613996360058, "end": 468.0055704700062, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 468.01859926600446, "end": 660.3070247620053, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 660.3070949360044, "end": 660.3509060910001, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 660.3661442240045, "end": 852.2493564700053, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 852.2494437210044, "end": 852.2910938360001, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 852.3061488890016, "end": 1054.6667658400038, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 1054.6668097660004, "end": 1054.7083348380038, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 1054.7219574340052, "end": 1245.6725224020047, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 1245.6726198280012, "end": 1245.7142320310013, "split": "test", "subset": "portuguese" } ] }