{ "dataset_revision": "56cef947981ee1263be298081ab126f9493a3065", "task_name": "Vidore3TelecomBM25sReranking.v2", "mteb_version": "2.24.9", "scores": { "test": [ { "ndcg_at_1": 0.66667, "ndcg_at_3": 0.61979, "ndcg_at_5": 0.61509, "ndcg_at_10": 0.63036, "ndcg_at_20": 0.64144, "ndcg_at_100": 0.64764, "ndcg_at_1000": 0.64764, "map_at_1": 0.29897, "map_at_3": 0.44479, "map_at_5": 0.48543, "map_at_10": 0.5286, "map_at_20": 0.54277, "map_at_100": 0.54685, "map_at_1000": 0.54685, "recall_at_1": 0.29897, "recall_at_3": 0.4746, "recall_at_5": 0.54421, "recall_at_10": 0.6393, "recall_at_20": 0.6837, "recall_at_100": 0.70164, "recall_at_1000": 0.70164, "accuracy": 0.29897, "precision_at_1": 0.68103, "precision_at_3": 0.45211, "precision_at_5": 0.33966, "precision_at_10": 0.22184, "precision_at_20": 0.12687, "precision_at_100": 0.02684, "precision_at_1000": 0.00268, "mrr_at_1": 0.681034, "mrr_at_3": 0.738985, "mrr_at_5": 0.745881, "mrr_at_10": 0.749616, "mrr_at_20": 0.750015, "mrr_at_100": 0.750121, "mrr_at_1000": 0.750121, "nauc_ndcg_at_1_max": 0.661068, "nauc_ndcg_at_1_std": 0.274918, "nauc_ndcg_at_1_diff1": 0.485795, "nauc_ndcg_at_3_max": 0.595158, "nauc_ndcg_at_3_std": 0.304517, "nauc_ndcg_at_3_diff1": 0.351927, "nauc_ndcg_at_5_max": 0.608259, "nauc_ndcg_at_5_std": 0.327488, "nauc_ndcg_at_5_diff1": 0.358399, "nauc_ndcg_at_10_max": 0.639361, "nauc_ndcg_at_10_std": 0.362154, "nauc_ndcg_at_10_diff1": 0.380771, "nauc_ndcg_at_20_max": 0.657663, "nauc_ndcg_at_20_std": 0.370911, "nauc_ndcg_at_20_diff1": 0.387042, "nauc_ndcg_at_100_max": 0.661844, "nauc_ndcg_at_100_std": 0.375289, "nauc_ndcg_at_100_diff1": 0.382282, "nauc_ndcg_at_1000_max": 0.661844, "nauc_ndcg_at_1000_std": 0.375289, "nauc_ndcg_at_1000_diff1": 0.382282, "nauc_map_at_1_max": 0.456848, "nauc_map_at_1_std": 0.021163, "nauc_map_at_1_diff1": 0.500968, "nauc_map_at_3_max": 0.507586, "nauc_map_at_3_std": 0.1495, "nauc_map_at_3_diff1": 0.369252, "nauc_map_at_5_max": 0.534321, "nauc_map_at_5_std": 0.203035, "nauc_map_at_5_diff1": 0.356207, "nauc_map_at_10_max": 0.563897, "nauc_map_at_10_std": 0.269831, "nauc_map_at_10_diff1": 0.356635, "nauc_map_at_20_max": 0.575672, "nauc_map_at_20_std": 0.278764, "nauc_map_at_20_diff1": 0.357481, "nauc_map_at_100_max": 0.577947, "nauc_map_at_100_std": 0.281729, "nauc_map_at_100_diff1": 0.354318, "nauc_map_at_1000_max": 0.577947, "nauc_map_at_1000_std": 0.281729, "nauc_map_at_1000_diff1": 0.354318, "nauc_recall_at_1_max": 0.456848, "nauc_recall_at_1_std": 0.021163, "nauc_recall_at_1_diff1": 0.500968, "nauc_recall_at_3_max": 0.494757, "nauc_recall_at_3_std": 0.168439, "nauc_recall_at_3_diff1": 0.333043, "nauc_recall_at_5_max": 0.53719, "nauc_recall_at_5_std": 0.26035, "nauc_recall_at_5_diff1": 0.319417, "nauc_recall_at_10_max": 0.604318, "nauc_recall_at_10_std": 0.38908, "nauc_recall_at_10_diff1": 0.325818, "nauc_recall_at_20_max": 0.645278, "nauc_recall_at_20_std": 0.44172, "nauc_recall_at_20_diff1": 0.325225, "nauc_recall_at_100_max": 0.656724, "nauc_recall_at_100_std": 0.452752, "nauc_recall_at_100_diff1": 0.305841, "nauc_recall_at_1000_max": 0.656724, "nauc_recall_at_1000_std": 0.452752, "nauc_recall_at_1000_diff1": 0.305841, "nauc_precision_at_1_max": 0.682696, "nauc_precision_at_1_std": 0.294392, "nauc_precision_at_1_diff1": 0.479687, "nauc_precision_at_3_max": 0.328443, "nauc_precision_at_3_std": 0.36027, "nauc_precision_at_3_diff1": -0.02466, "nauc_precision_at_5_max": 0.231064, "nauc_precision_at_5_std": 0.387749, "nauc_precision_at_5_diff1": -0.080764, "nauc_precision_at_10_max": 0.151412, "nauc_precision_at_10_std": 0.384754, "nauc_precision_at_10_diff1": -0.091025, "nauc_precision_at_20_max": 0.115165, "nauc_precision_at_20_std": 0.33103, "nauc_precision_at_20_diff1": -0.108508, "nauc_precision_at_100_max": 0.0961, "nauc_precision_at_100_std": 0.302786, "nauc_precision_at_100_diff1": -0.125711, "nauc_precision_at_1000_max": 0.0961, "nauc_precision_at_1000_std": 0.302786, "nauc_precision_at_1000_diff1": -0.125711, "nauc_mrr_at_1_max": 0.682696, "nauc_mrr_at_1_std": 0.294392, "nauc_mrr_at_1_diff1": 0.479687, "nauc_mrr_at_3_max": 0.719941, "nauc_mrr_at_3_std": 0.374705, "nauc_mrr_at_3_diff1": 0.446905, "nauc_mrr_at_5_max": 0.726971, "nauc_mrr_at_5_std": 0.387421, "nauc_mrr_at_5_diff1": 0.453962, "nauc_mrr_at_10_max": 0.732277, "nauc_mrr_at_10_std": 0.393312, "nauc_mrr_at_10_diff1": 0.456878, "nauc_mrr_at_20_max": 0.732226, "nauc_mrr_at_20_std": 0.39645, "nauc_mrr_at_20_diff1": 0.456735, "nauc_mrr_at_100_max": 0.732184, "nauc_mrr_at_100_std": 0.396196, "nauc_mrr_at_100_diff1": 0.456577, "nauc_mrr_at_1000_max": 0.732184, "nauc_mrr_at_1000_std": 0.396196, "nauc_mrr_at_1000_diff1": 0.456577, "hit_rate_at_1": 0.68103, "hit_rate_at_3": 0.80747, "hit_rate_at_5": 0.83621, "hit_rate_at_10": 0.86207, "hit_rate_at_20": 0.86782, "hit_rate_at_100": 0.87069, "hit_rate_at_1000": 0.87069, "main_score": 0.63036, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.625, "ndcg_at_3": 0.57424, "ndcg_at_5": 0.57405, "ndcg_at_10": 0.58499, "ndcg_at_20": 0.59722, "ndcg_at_100": 0.60338, "ndcg_at_1000": 0.60338, "map_at_1": 0.2784, "map_at_3": 0.41124, "map_at_5": 0.45265, "map_at_10": 0.49193, "map_at_20": 0.50695, "map_at_100": 0.51117, "map_at_1000": 0.51117, "recall_at_1": 0.2784, "recall_at_3": 0.43972, "recall_at_5": 0.5096, "recall_at_10": 0.59014, "recall_at_20": 0.63816, "recall_at_100": 0.65391, "recall_at_1000": 0.65391, "accuracy": 0.2784, "precision_at_1": 0.64368, "precision_at_3": 0.41571, "precision_at_5": 0.31667, "precision_at_10": 0.20747, "precision_at_20": 0.1204, "precision_at_100": 0.02583, "precision_at_1000": 0.00258, "mrr_at_1": 0.643678, "mrr_at_3": 0.686782, "mrr_at_5": 0.693391, "mrr_at_10": 0.696295, "mrr_at_20": 0.697302, "mrr_at_100": 0.697377, "mrr_at_1000": 0.697377, "nauc_ndcg_at_1_max": 0.690867, "nauc_ndcg_at_1_std": 0.3885, "nauc_ndcg_at_1_diff1": 0.483005, "nauc_ndcg_at_3_max": 0.645624, "nauc_ndcg_at_3_std": 0.3663, "nauc_ndcg_at_3_diff1": 0.414853, "nauc_ndcg_at_5_max": 0.660039, "nauc_ndcg_at_5_std": 0.386467, "nauc_ndcg_at_5_diff1": 0.407263, "nauc_ndcg_at_10_max": 0.681548, "nauc_ndcg_at_10_std": 0.416309, "nauc_ndcg_at_10_diff1": 0.424865, "nauc_ndcg_at_20_max": 0.69865, "nauc_ndcg_at_20_std": 0.443568, "nauc_ndcg_at_20_diff1": 0.424181, "nauc_ndcg_at_100_max": 0.701421, "nauc_ndcg_at_100_std": 0.45154, "nauc_ndcg_at_100_diff1": 0.424807, "nauc_ndcg_at_1000_max": 0.701421, "nauc_ndcg_at_1000_std": 0.45154, "nauc_ndcg_at_1000_diff1": 0.424807, "nauc_map_at_1_max": 0.48072, "nauc_map_at_1_std": 0.067099, "nauc_map_at_1_diff1": 0.54145, "nauc_map_at_3_max": 0.532867, "nauc_map_at_3_std": 0.187866, "nauc_map_at_3_diff1": 0.421553, "nauc_map_at_5_max": 0.572826, "nauc_map_at_5_std": 0.242469, "nauc_map_at_5_diff1": 0.408229, "nauc_map_at_10_max": 0.604419, "nauc_map_at_10_std": 0.315094, "nauc_map_at_10_diff1": 0.405062, "nauc_map_at_20_max": 0.61988, "nauc_map_at_20_std": 0.340788, "nauc_map_at_20_diff1": 0.403176, "nauc_map_at_100_max": 0.621642, "nauc_map_at_100_std": 0.347598, "nauc_map_at_100_diff1": 0.402952, "nauc_map_at_1000_max": 0.621642, "nauc_map_at_1000_std": 0.347598, "nauc_map_at_1000_diff1": 0.402952, "nauc_recall_at_1_max": 0.48072, "nauc_recall_at_1_std": 0.067099, "nauc_recall_at_1_diff1": 0.54145, "nauc_recall_at_3_max": 0.517505, "nauc_recall_at_3_std": 0.198305, "nauc_recall_at_3_diff1": 0.38387, "nauc_recall_at_5_max": 0.565588, "nauc_recall_at_5_std": 0.289905, "nauc_recall_at_5_diff1": 0.354858, "nauc_recall_at_10_max": 0.637071, "nauc_recall_at_10_std": 0.413637, "nauc_recall_at_10_diff1": 0.367183, "nauc_recall_at_20_max": 0.681609, "nauc_recall_at_20_std": 0.503106, "nauc_recall_at_20_diff1": 0.359441, "nauc_recall_at_100_max": 0.689662, "nauc_recall_at_100_std": 0.521431, "nauc_recall_at_100_diff1": 0.359682, "nauc_recall_at_1000_max": 0.689662, "nauc_recall_at_1000_std": 0.521431, "nauc_recall_at_1000_diff1": 0.359682, "nauc_precision_at_1_max": 0.715778, "nauc_precision_at_1_std": 0.421381, "nauc_precision_at_1_diff1": 0.493964, "nauc_precision_at_3_max": 0.423997, "nauc_precision_at_3_std": 0.406694, "nauc_precision_at_3_diff1": 0.081295, "nauc_precision_at_5_max": 0.345159, "nauc_precision_at_5_std": 0.424233, "nauc_precision_at_5_diff1": -0.001576, "nauc_precision_at_10_max": 0.259177, "nauc_precision_at_10_std": 0.422588, "nauc_precision_at_10_diff1": -0.022654, "nauc_precision_at_20_max": 0.217886, "nauc_precision_at_20_std": 0.413999, "nauc_precision_at_20_diff1": -0.053845, "nauc_precision_at_100_max": 0.17839, "nauc_precision_at_100_std": 0.383124, "nauc_precision_at_100_diff1": -0.059545, "nauc_precision_at_1000_max": 0.17839, "nauc_precision_at_1000_std": 0.383124, "nauc_precision_at_1000_diff1": -0.059545, "nauc_mrr_at_1_max": 0.715778, "nauc_mrr_at_1_std": 0.421381, "nauc_mrr_at_1_diff1": 0.493964, "nauc_mrr_at_3_max": 0.747246, "nauc_mrr_at_3_std": 0.494267, "nauc_mrr_at_3_diff1": 0.469604, "nauc_mrr_at_5_max": 0.753567, "nauc_mrr_at_5_std": 0.514961, "nauc_mrr_at_5_diff1": 0.468516, "nauc_mrr_at_10_max": 0.757602, "nauc_mrr_at_10_std": 0.515195, "nauc_mrr_at_10_diff1": 0.478308, "nauc_mrr_at_20_max": 0.757814, "nauc_mrr_at_20_std": 0.516142, "nauc_mrr_at_20_diff1": 0.477439, "nauc_mrr_at_100_max": 0.757796, "nauc_mrr_at_100_std": 0.516062, "nauc_mrr_at_100_diff1": 0.477305, "nauc_mrr_at_1000_max": 0.757796, "nauc_mrr_at_1000_std": 0.516062, "nauc_mrr_at_1000_diff1": 0.477305, "hit_rate_at_1": 0.64368, "hit_rate_at_3": 0.73851, "hit_rate_at_5": 0.76724, "hit_rate_at_10": 0.78736, "hit_rate_at_20": 0.80172, "hit_rate_at_100": 0.8046, "hit_rate_at_1000": 0.8046, "main_score": 0.58499, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.73563, "ndcg_at_3": 0.69923, "ndcg_at_5": 0.70349, "ndcg_at_10": 0.72235, "ndcg_at_20": 0.74499, "ndcg_at_100": 0.75359, "ndcg_at_1000": 0.75359, "map_at_1": 0.33506, "map_at_3": 0.50937, "map_at_5": 0.56487, "map_at_10": 0.61408, "map_at_20": 0.63592, "map_at_100": 0.64146, "map_at_1000": 0.64146, "recall_at_1": 0.33506, "recall_at_3": 0.5419, "recall_at_5": 0.636, "recall_at_10": 0.74235, "recall_at_20": 0.81743, "recall_at_100": 0.84072, "recall_at_1000": 0.84072, "accuracy": 0.33506, "precision_at_1": 0.75575, "precision_at_3": 0.51628, "precision_at_5": 0.39828, "precision_at_10": 0.25747, "precision_at_20": 0.15359, "precision_at_100": 0.03279, "precision_at_1000": 0.00328, "mrr_at_1": 0.755747, "mrr_at_3": 0.818966, "mrr_at_5": 0.828017, "mrr_at_10": 0.831999, "mrr_at_20": 0.833617, "mrr_at_100": 0.833617, "mrr_at_1000": 0.833617, "nauc_ndcg_at_1_max": 0.466048, "nauc_ndcg_at_1_std": -0.090567, "nauc_ndcg_at_1_diff1": 0.446497, "nauc_ndcg_at_3_max": 0.488414, "nauc_ndcg_at_3_std": 0.011432, "nauc_ndcg_at_3_diff1": 0.290853, "nauc_ndcg_at_5_max": 0.474602, "nauc_ndcg_at_5_std": 0.023967, "nauc_ndcg_at_5_diff1": 0.289535, "nauc_ndcg_at_10_max": 0.492811, "nauc_ndcg_at_10_std": 0.036969, "nauc_ndcg_at_10_diff1": 0.306316, "nauc_ndcg_at_20_max": 0.512492, "nauc_ndcg_at_20_std": 0.067167, "nauc_ndcg_at_20_diff1": 0.300362, "nauc_ndcg_at_100_max": 0.511354, "nauc_ndcg_at_100_std": 0.06038, "nauc_ndcg_at_100_diff1": 0.287443, "nauc_ndcg_at_1000_max": 0.511354, "nauc_ndcg_at_1000_std": 0.06038, "nauc_ndcg_at_1000_diff1": 0.287443, "nauc_map_at_1_max": 0.357973, "nauc_map_at_1_std": -0.158416, "nauc_map_at_1_diff1": 0.488904, "nauc_map_at_3_max": 0.42522, "nauc_map_at_3_std": -0.079161, "nauc_map_at_3_diff1": 0.333624, "nauc_map_at_5_max": 0.42714, "nauc_map_at_5_std": -0.047889, "nauc_map_at_5_diff1": 0.303696, "nauc_map_at_10_max": 0.438122, "nauc_map_at_10_std": 0.006824, "nauc_map_at_10_diff1": 0.287422, "nauc_map_at_20_max": 0.44946, "nauc_map_at_20_std": 0.0285, "nauc_map_at_20_diff1": 0.28159, "nauc_map_at_100_max": 0.449486, "nauc_map_at_100_std": 0.025502, "nauc_map_at_100_diff1": 0.275199, "nauc_map_at_1000_max": 0.449486, "nauc_map_at_1000_std": 0.025502, "nauc_map_at_1000_diff1": 0.275199, "nauc_recall_at_1_max": 0.357973, "nauc_recall_at_1_std": -0.158416, "nauc_recall_at_1_diff1": 0.488904, "nauc_recall_at_3_max": 0.430695, "nauc_recall_at_3_std": -0.058707, "nauc_recall_at_3_diff1": 0.292816, "nauc_recall_at_5_max": 0.409346, "nauc_recall_at_5_std": -0.007664, "nauc_recall_at_5_diff1": 0.239417, "nauc_recall_at_10_max": 0.444184, "nauc_recall_at_10_std": 0.090218, "nauc_recall_at_10_diff1": 0.215173, "nauc_recall_at_20_max": 0.485513, "nauc_recall_at_20_std": 0.236491, "nauc_recall_at_20_diff1": 0.169118, "nauc_recall_at_100_max": 0.489271, "nauc_recall_at_100_std": 0.234734, "nauc_recall_at_100_diff1": 0.111783, "nauc_recall_at_1000_max": 0.489271, "nauc_recall_at_1000_std": 0.234734, "nauc_recall_at_1000_diff1": 0.111783, "nauc_precision_at_1_max": 0.470699, "nauc_precision_at_1_std": -0.098501, "nauc_precision_at_1_diff1": 0.447686, "nauc_precision_at_3_max": 0.182829, "nauc_precision_at_3_std": 0.153961, "nauc_precision_at_3_diff1": -0.13731, "nauc_precision_at_5_max": 0.034273, "nauc_precision_at_5_std": 0.194397, "nauc_precision_at_5_diff1": -0.212464, "nauc_precision_at_10_max": -0.041032, "nauc_precision_at_10_std": 0.236132, "nauc_precision_at_10_diff1": -0.216835, "nauc_precision_at_20_max": -0.090263, "nauc_precision_at_20_std": 0.225327, "nauc_precision_at_20_diff1": -0.239076, "nauc_precision_at_100_max": -0.114563, "nauc_precision_at_100_std": 0.177367, "nauc_precision_at_100_diff1": -0.25972, "nauc_precision_at_1000_max": -0.114563, "nauc_precision_at_1000_std": 0.177367, "nauc_precision_at_1000_diff1": -0.25972, "nauc_mrr_at_1_max": 0.470699, "nauc_mrr_at_1_std": -0.098501, "nauc_mrr_at_1_diff1": 0.447686, "nauc_mrr_at_3_max": 0.521037, "nauc_mrr_at_3_std": -0.029987, "nauc_mrr_at_3_diff1": 0.380442, "nauc_mrr_at_5_max": 0.523306, "nauc_mrr_at_5_std": -0.038371, "nauc_mrr_at_5_diff1": 0.397704, "nauc_mrr_at_10_max": 0.527426, "nauc_mrr_at_10_std": -0.03098, "nauc_mrr_at_10_diff1": 0.39861, "nauc_mrr_at_20_max": 0.524096, "nauc_mrr_at_20_std": -0.029109, "nauc_mrr_at_20_diff1": 0.39761, "nauc_mrr_at_100_max": 0.524096, "nauc_mrr_at_100_std": -0.029109, "nauc_mrr_at_100_diff1": 0.39761, "nauc_mrr_at_1000_max": 0.524096, "nauc_mrr_at_1000_std": -0.029109, "nauc_mrr_at_1000_diff1": 0.39761, "hit_rate_at_1": 0.75575, "hit_rate_at_3": 0.88793, "hit_rate_at_5": 0.92529, "hit_rate_at_10": 0.95115, "hit_rate_at_20": 0.97414, "hit_rate_at_100": 0.97414, "hit_rate_at_1000": 0.97414, "main_score": 0.72235, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.66954, "ndcg_at_3": 0.60646, "ndcg_at_5": 0.60952, "ndcg_at_10": 0.62133, "ndcg_at_20": 0.63654, "ndcg_at_100": 0.64648, "ndcg_at_1000": 0.64648, "map_at_1": 0.2971, "map_at_3": 0.43335, "map_at_5": 0.4844, "map_at_10": 0.52353, "map_at_20": 0.54084, "map_at_100": 0.54688, "map_at_1000": 0.54688, "recall_at_1": 0.2971, "recall_at_3": 0.45989, "recall_at_5": 0.54769, "recall_at_10": 0.62999, "recall_at_20": 0.67946, "recall_at_100": 0.70682, "recall_at_1000": 0.70682, "accuracy": 0.2971, "precision_at_1": 0.68391, "precision_at_3": 0.43678, "precision_at_5": 0.33678, "precision_at_10": 0.21667, "precision_at_20": 0.12672, "precision_at_100": 0.02756, "precision_at_1000": 0.00276, "mrr_at_1": 0.683908, "mrr_at_3": 0.732759, "mrr_at_5": 0.737069, "mrr_at_10": 0.7408, "mrr_at_20": 0.741332, "mrr_at_100": 0.741589, "mrr_at_1000": 0.741589, "nauc_ndcg_at_1_max": 0.655226, "nauc_ndcg_at_1_std": 0.386212, "nauc_ndcg_at_1_diff1": 0.378114, "nauc_ndcg_at_3_max": 0.6372, "nauc_ndcg_at_3_std": 0.357687, "nauc_ndcg_at_3_diff1": 0.322434, "nauc_ndcg_at_5_max": 0.651996, "nauc_ndcg_at_5_std": 0.391443, "nauc_ndcg_at_5_diff1": 0.339404, "nauc_ndcg_at_10_max": 0.6716, "nauc_ndcg_at_10_std": 0.423899, "nauc_ndcg_at_10_diff1": 0.361132, "nauc_ndcg_at_20_max": 0.689393, "nauc_ndcg_at_20_std": 0.457964, "nauc_ndcg_at_20_diff1": 0.362388, "nauc_ndcg_at_100_max": 0.691762, "nauc_ndcg_at_100_std": 0.461362, "nauc_ndcg_at_100_diff1": 0.360115, "nauc_ndcg_at_1000_max": 0.691762, "nauc_ndcg_at_1000_std": 0.461362, "nauc_ndcg_at_1000_diff1": 0.360115, "nauc_map_at_1_max": 0.474685, "nauc_map_at_1_std": 0.056094, "nauc_map_at_1_diff1": 0.503824, "nauc_map_at_3_max": 0.548095, "nauc_map_at_3_std": 0.185487, "nauc_map_at_3_diff1": 0.374962, "nauc_map_at_5_max": 0.578452, "nauc_map_at_5_std": 0.25044, "nauc_map_at_5_diff1": 0.366286, "nauc_map_at_10_max": 0.601283, "nauc_map_at_10_std": 0.319315, "nauc_map_at_10_diff1": 0.362045, "nauc_map_at_20_max": 0.614214, "nauc_map_at_20_std": 0.348061, "nauc_map_at_20_diff1": 0.357945, "nauc_map_at_100_max": 0.615248, "nauc_map_at_100_std": 0.351666, "nauc_map_at_100_diff1": 0.35613, "nauc_map_at_1000_max": 0.615248, "nauc_map_at_1000_std": 0.351666, "nauc_map_at_1000_diff1": 0.35613, "nauc_recall_at_1_max": 0.474685, "nauc_recall_at_1_std": 0.056094, "nauc_recall_at_1_diff1": 0.503824, "nauc_recall_at_3_max": 0.544302, "nauc_recall_at_3_std": 0.201906, "nauc_recall_at_3_diff1": 0.353043, "nauc_recall_at_5_max": 0.576754, "nauc_recall_at_5_std": 0.312977, "nauc_recall_at_5_diff1": 0.350947, "nauc_recall_at_10_max": 0.643179, "nauc_recall_at_10_std": 0.448225, "nauc_recall_at_10_diff1": 0.359378, "nauc_recall_at_20_max": 0.689242, "nauc_recall_at_20_std": 0.5464, "nauc_recall_at_20_diff1": 0.35705, "nauc_recall_at_100_max": 0.699948, "nauc_recall_at_100_std": 0.560983, "nauc_recall_at_100_diff1": 0.347778, "nauc_recall_at_1000_max": 0.699948, "nauc_recall_at_1000_std": 0.560983, "nauc_recall_at_1000_diff1": 0.347778, "nauc_precision_at_1_max": 0.665804, "nauc_precision_at_1_std": 0.39745, "nauc_precision_at_1_diff1": 0.385971, "nauc_precision_at_3_max": 0.385909, "nauc_precision_at_3_std": 0.400792, "nauc_precision_at_3_diff1": -0.033395, "nauc_precision_at_5_max": 0.291562, "nauc_precision_at_5_std": 0.423712, "nauc_precision_at_5_diff1": -0.072589, "nauc_precision_at_10_max": 0.180246, "nauc_precision_at_10_std": 0.410454, "nauc_precision_at_10_diff1": -0.109919, "nauc_precision_at_20_max": 0.127372, "nauc_precision_at_20_std": 0.373697, "nauc_precision_at_20_diff1": -0.135007, "nauc_precision_at_100_max": 0.087389, "nauc_precision_at_100_std": 0.322103, "nauc_precision_at_100_diff1": -0.141958, "nauc_precision_at_1000_max": 0.087389, "nauc_precision_at_1000_std": 0.322103, "nauc_precision_at_1000_diff1": -0.141958, "nauc_mrr_at_1_max": 0.665804, "nauc_mrr_at_1_std": 0.39745, "nauc_mrr_at_1_diff1": 0.385971, "nauc_mrr_at_3_max": 0.724465, "nauc_mrr_at_3_std": 0.477584, "nauc_mrr_at_3_diff1": 0.367315, "nauc_mrr_at_5_max": 0.728813, "nauc_mrr_at_5_std": 0.485408, "nauc_mrr_at_5_diff1": 0.371035, "nauc_mrr_at_10_max": 0.73309, "nauc_mrr_at_10_std": 0.493834, "nauc_mrr_at_10_diff1": 0.374897, "nauc_mrr_at_20_max": 0.73286, "nauc_mrr_at_20_std": 0.495256, "nauc_mrr_at_20_diff1": 0.374363, "nauc_mrr_at_100_max": 0.732954, "nauc_mrr_at_100_std": 0.495117, "nauc_mrr_at_100_diff1": 0.374386, "nauc_mrr_at_1000_max": 0.732954, "nauc_mrr_at_1000_std": 0.495117, "nauc_mrr_at_1000_diff1": 0.374386, "hit_rate_at_1": 0.68391, "hit_rate_at_3": 0.79023, "hit_rate_at_5": 0.80747, "hit_rate_at_10": 0.83333, "hit_rate_at_20": 0.84195, "hit_rate_at_100": 0.8477, "hit_rate_at_1000": 0.8477, "main_score": 0.62133, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.65517, "ndcg_at_3": 0.60169, "ndcg_at_5": 0.59868, "ndcg_at_10": 0.613, "ndcg_at_20": 0.62337, "ndcg_at_100": 0.63073, "ndcg_at_1000": 0.63073, "map_at_1": 0.30007, "map_at_3": 0.43832, "map_at_5": 0.47804, "map_at_10": 0.51799, "map_at_20": 0.5315, "map_at_100": 0.53603, "map_at_1000": 0.53603, "recall_at_1": 0.30007, "recall_at_3": 0.46733, "recall_at_5": 0.53495, "recall_at_10": 0.62382, "recall_at_20": 0.66537, "recall_at_100": 0.68705, "recall_at_1000": 0.68705, "accuracy": 0.30007, "precision_at_1": 0.67529, "precision_at_3": 0.43487, "precision_at_5": 0.32529, "precision_at_10": 0.21149, "precision_at_20": 0.12026, "precision_at_100": 0.02572, "precision_at_1000": 0.00257, "mrr_at_1": 0.675287, "mrr_at_3": 0.72318, "mrr_at_5": 0.729789, "mrr_at_10": 0.732447, "mrr_at_20": 0.732778, "mrr_at_100": 0.732881, "mrr_at_1000": 0.732881, "nauc_ndcg_at_1_max": 0.646002, "nauc_ndcg_at_1_std": 0.300509, "nauc_ndcg_at_1_diff1": 0.350349, "nauc_ndcg_at_3_max": 0.620132, "nauc_ndcg_at_3_std": 0.335712, "nauc_ndcg_at_3_diff1": 0.295156, "nauc_ndcg_at_5_max": 0.637977, "nauc_ndcg_at_5_std": 0.374483, "nauc_ndcg_at_5_diff1": 0.290883, "nauc_ndcg_at_10_max": 0.662047, "nauc_ndcg_at_10_std": 0.398435, "nauc_ndcg_at_10_diff1": 0.305142, "nauc_ndcg_at_20_max": 0.682045, "nauc_ndcg_at_20_std": 0.41909, "nauc_ndcg_at_20_diff1": 0.301341, "nauc_ndcg_at_100_max": 0.686974, "nauc_ndcg_at_100_std": 0.428334, "nauc_ndcg_at_100_diff1": 0.298694, "nauc_ndcg_at_1000_max": 0.686974, "nauc_ndcg_at_1000_std": 0.428334, "nauc_ndcg_at_1000_diff1": 0.298694, "nauc_map_at_1_max": 0.479989, "nauc_map_at_1_std": 0.042751, "nauc_map_at_1_diff1": 0.447098, "nauc_map_at_3_max": 0.529598, "nauc_map_at_3_std": 0.182076, "nauc_map_at_3_diff1": 0.327688, "nauc_map_at_5_max": 0.562726, "nauc_map_at_5_std": 0.245801, "nauc_map_at_5_diff1": 0.302545, "nauc_map_at_10_max": 0.590107, "nauc_map_at_10_std": 0.297393, "nauc_map_at_10_diff1": 0.300444, "nauc_map_at_20_max": 0.606442, "nauc_map_at_20_std": 0.320918, "nauc_map_at_20_diff1": 0.298015, "nauc_map_at_100_max": 0.610198, "nauc_map_at_100_std": 0.32678, "nauc_map_at_100_diff1": 0.297696, "nauc_map_at_1000_max": 0.610198, "nauc_map_at_1000_std": 0.32678, "nauc_map_at_1000_diff1": 0.297696, "nauc_recall_at_1_max": 0.479989, "nauc_recall_at_1_std": 0.042751, "nauc_recall_at_1_diff1": 0.447098, "nauc_recall_at_3_max": 0.52536, "nauc_recall_at_3_std": 0.223277, "nauc_recall_at_3_diff1": 0.299531, "nauc_recall_at_5_max": 0.568412, "nauc_recall_at_5_std": 0.323269, "nauc_recall_at_5_diff1": 0.264403, "nauc_recall_at_10_max": 0.639553, "nauc_recall_at_10_std": 0.428962, "nauc_recall_at_10_diff1": 0.261618, "nauc_recall_at_20_max": 0.693286, "nauc_recall_at_20_std": 0.51766, "nauc_recall_at_20_diff1": 0.241627, "nauc_recall_at_100_max": 0.709489, "nauc_recall_at_100_std": 0.54826, "nauc_recall_at_100_diff1": 0.225755, "nauc_recall_at_1000_max": 0.709489, "nauc_recall_at_1000_std": 0.54826, "nauc_recall_at_1000_diff1": 0.225755, "nauc_precision_at_1_max": 0.664693, "nauc_precision_at_1_std": 0.331781, "nauc_precision_at_1_diff1": 0.362208, "nauc_precision_at_3_max": 0.37239, "nauc_precision_at_3_std": 0.414737, "nauc_precision_at_3_diff1": -0.030362, "nauc_precision_at_5_max": 0.292166, "nauc_precision_at_5_std": 0.437686, "nauc_precision_at_5_diff1": -0.100975, "nauc_precision_at_10_max": 0.195355, "nauc_precision_at_10_std": 0.396639, "nauc_precision_at_10_diff1": -0.098719, "nauc_precision_at_20_max": 0.1606, "nauc_precision_at_20_std": 0.387256, "nauc_precision_at_20_diff1": -0.113926, "nauc_precision_at_100_max": 0.135057, "nauc_precision_at_100_std": 0.361236, "nauc_precision_at_100_diff1": -0.115196, "nauc_precision_at_1000_max": 0.135057, "nauc_precision_at_1000_std": 0.361236, "nauc_precision_at_1000_diff1": -0.115196, "nauc_mrr_at_1_max": 0.664693, "nauc_mrr_at_1_std": 0.331781, "nauc_mrr_at_1_diff1": 0.362208, "nauc_mrr_at_3_max": 0.703421, "nauc_mrr_at_3_std": 0.432057, "nauc_mrr_at_3_diff1": 0.347751, "nauc_mrr_at_5_max": 0.716161, "nauc_mrr_at_5_std": 0.447527, "nauc_mrr_at_5_diff1": 0.346048, "nauc_mrr_at_10_max": 0.720892, "nauc_mrr_at_10_std": 0.450714, "nauc_mrr_at_10_diff1": 0.348573, "nauc_mrr_at_20_max": 0.721122, "nauc_mrr_at_20_std": 0.452122, "nauc_mrr_at_20_diff1": 0.348086, "nauc_mrr_at_100_max": 0.721154, "nauc_mrr_at_100_std": 0.452051, "nauc_mrr_at_100_diff1": 0.348061, "nauc_mrr_at_1000_max": 0.721154, "nauc_mrr_at_1000_std": 0.452051, "nauc_mrr_at_1000_diff1": 0.348061, "hit_rate_at_1": 0.67529, "hit_rate_at_3": 0.78448, "hit_rate_at_5": 0.81322, "hit_rate_at_10": 0.83621, "hit_rate_at_20": 0.84195, "hit_rate_at_100": 0.84483, "hit_rate_at_1000": 0.84483, "main_score": 0.613, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.60345, "ndcg_at_3": 0.57307, "ndcg_at_5": 0.57724, "ndcg_at_10": 0.59058, "ndcg_at_20": 0.60332, "ndcg_at_100": 0.6079, "ndcg_at_1000": 0.6079, "map_at_1": 0.27591, "map_at_3": 0.41525, "map_at_5": 0.45946, "map_at_10": 0.49869, "map_at_20": 0.5139, "map_at_100": 0.51692, "map_at_1000": 0.51692, "recall_at_1": 0.27591, "recall_at_3": 0.45024, "recall_at_5": 0.52418, "recall_at_10": 0.60415, "recall_at_20": 0.65084, "recall_at_100": 0.66284, "recall_at_1000": 0.66284, "accuracy": 0.27591, "precision_at_1": 0.62069, "precision_at_3": 0.41379, "precision_at_5": 0.31839, "precision_at_10": 0.20776, "precision_at_20": 0.1194, "precision_at_100": 0.02509, "precision_at_1000": 0.00251, "mrr_at_1": 0.62069, "mrr_at_3": 0.678161, "mrr_at_5": 0.684626, "mrr_at_10": 0.688042, "mrr_at_20": 0.688918, "mrr_at_100": 0.688918, "mrr_at_1000": 0.688918, "nauc_ndcg_at_1_max": 0.676383, "nauc_ndcg_at_1_std": 0.330165, "nauc_ndcg_at_1_diff1": 0.510141, "nauc_ndcg_at_3_max": 0.649515, "nauc_ndcg_at_3_std": 0.377597, "nauc_ndcg_at_3_diff1": 0.408035, "nauc_ndcg_at_5_max": 0.664683, "nauc_ndcg_at_5_std": 0.413118, "nauc_ndcg_at_5_diff1": 0.401274, "nauc_ndcg_at_10_max": 0.681756, "nauc_ndcg_at_10_std": 0.420838, "nauc_ndcg_at_10_diff1": 0.42325, "nauc_ndcg_at_20_max": 0.699099, "nauc_ndcg_at_20_std": 0.449045, "nauc_ndcg_at_20_diff1": 0.423415, "nauc_ndcg_at_100_max": 0.703205, "nauc_ndcg_at_100_std": 0.45225, "nauc_ndcg_at_100_diff1": 0.427687, "nauc_ndcg_at_1000_max": 0.703205, "nauc_ndcg_at_1000_std": 0.45225, "nauc_ndcg_at_1000_diff1": 0.427687, "nauc_map_at_1_max": 0.471203, "nauc_map_at_1_std": 0.052372, "nauc_map_at_1_diff1": 0.565798, "nauc_map_at_3_max": 0.524919, "nauc_map_at_3_std": 0.183845, "nauc_map_at_3_diff1": 0.416347, "nauc_map_at_5_max": 0.565325, "nauc_map_at_5_std": 0.258866, "nauc_map_at_5_diff1": 0.398001, "nauc_map_at_10_max": 0.597194, "nauc_map_at_10_std": 0.320032, "nauc_map_at_10_diff1": 0.399032, "nauc_map_at_20_max": 0.612256, "nauc_map_at_20_std": 0.348123, "nauc_map_at_20_diff1": 0.396368, "nauc_map_at_100_max": 0.614139, "nauc_map_at_100_std": 0.351316, "nauc_map_at_100_diff1": 0.39797, "nauc_map_at_1000_max": 0.614139, "nauc_map_at_1000_std": 0.351316, "nauc_map_at_1000_diff1": 0.39797, "nauc_recall_at_1_max": 0.471203, "nauc_recall_at_1_std": 0.052372, "nauc_recall_at_1_diff1": 0.565798, "nauc_recall_at_3_max": 0.507137, "nauc_recall_at_3_std": 0.213278, "nauc_recall_at_3_diff1": 0.368999, "nauc_recall_at_5_max": 0.572955, "nauc_recall_at_5_std": 0.329418, "nauc_recall_at_5_diff1": 0.3488, "nauc_recall_at_10_max": 0.646904, "nauc_recall_at_10_std": 0.425818, "nauc_recall_at_10_diff1": 0.371516, "nauc_recall_at_20_max": 0.699372, "nauc_recall_at_20_std": 0.519692, "nauc_recall_at_20_diff1": 0.368252, "nauc_recall_at_100_max": 0.711027, "nauc_recall_at_100_std": 0.528325, "nauc_recall_at_100_diff1": 0.375826, "nauc_recall_at_1000_max": 0.711027, "nauc_recall_at_1000_std": 0.528325, "nauc_recall_at_1000_diff1": 0.375826, "nauc_precision_at_1_max": 0.69129, "nauc_precision_at_1_std": 0.348675, "nauc_precision_at_1_diff1": 0.515621, "nauc_precision_at_3_max": 0.450914, "nauc_precision_at_3_std": 0.4359, "nauc_precision_at_3_diff1": 0.054497, "nauc_precision_at_5_max": 0.385257, "nauc_precision_at_5_std": 0.495418, "nauc_precision_at_5_diff1": -0.026052, "nauc_precision_at_10_max": 0.282683, "nauc_precision_at_10_std": 0.453569, "nauc_precision_at_10_diff1": -0.035203, "nauc_precision_at_20_max": 0.241823, "nauc_precision_at_20_std": 0.441892, "nauc_precision_at_20_diff1": -0.062296, "nauc_precision_at_100_max": 0.217403, "nauc_precision_at_100_std": 0.41385, "nauc_precision_at_100_diff1": -0.056079, "nauc_precision_at_1000_max": 0.217403, "nauc_precision_at_1000_std": 0.41385, "nauc_precision_at_1000_diff1": -0.056079, "nauc_mrr_at_1_max": 0.69129, "nauc_mrr_at_1_std": 0.348675, "nauc_mrr_at_1_diff1": 0.515621, "nauc_mrr_at_3_max": 0.750808, "nauc_mrr_at_3_std": 0.457672, "nauc_mrr_at_3_diff1": 0.489361, "nauc_mrr_at_5_max": 0.7557, "nauc_mrr_at_5_std": 0.465796, "nauc_mrr_at_5_diff1": 0.48947, "nauc_mrr_at_10_max": 0.758479, "nauc_mrr_at_10_std": 0.469827, "nauc_mrr_at_10_diff1": 0.49262, "nauc_mrr_at_20_max": 0.758712, "nauc_mrr_at_20_std": 0.472099, "nauc_mrr_at_20_diff1": 0.492522, "nauc_mrr_at_100_max": 0.758712, "nauc_mrr_at_100_std": 0.472099, "nauc_mrr_at_100_diff1": 0.492522, "nauc_mrr_at_1000_max": 0.758712, "nauc_mrr_at_1000_std": 0.472099, "nauc_mrr_at_1000_diff1": 0.492522, "hit_rate_at_1": 0.62069, "hit_rate_at_3": 0.74713, "hit_rate_at_5": 0.77299, "hit_rate_at_10": 0.79598, "hit_rate_at_20": 0.81034, "hit_rate_at_100": 0.81034, "hit_rate_at_1000": 0.81034, "main_score": 0.59058, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.24.9" } ] }, "evaluation_time": 9168.382248749025, "kg_co2_emissions": null, "date": 1791974237.386518, "evaluation_phases": [ { "name": "Data loading", "start": 9.284005500376225e-06, "end": 32.50086505099898, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 32.500899732986, "end": 32.500902567000594, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 32.505838082986884, "end": 1586.1522610089742, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 1586.1523080400075, "end": 1586.202565376996, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 1586.214842786023, "end": 3116.727743090014, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 3116.727775018022, "end": 3116.772015937022, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 3116.7853656759835, "end": 4597.394653327996, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 4597.394683043007, "end": 4597.436887491029, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 4597.449066934991, "end": 6123.129965470987, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 6123.129996097006, "end": 6123.17220901401, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 6123.1851307179895, "end": 7631.435496750986, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 7631.435542219027, "end": 7631.47817079298, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 7631.497439233994, "end": 9168.329170464014, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 9168.32920233201, "end": 9168.372300362971, "split": "test", "subset": "portuguese" } ] }