{ "dataset_revision": "6d20dfc6799e48a4b1d9dab909980c06b9caa3d3", "task_name": "Vidore3PharmaceuticalsBM25sReranking.v2", "mteb_version": "2.24.9", "scores": { "test": [ { "ndcg_at_1": 0.50137, "ndcg_at_3": 0.49882, "ndcg_at_5": 0.49854, "ndcg_at_10": 0.5029, "ndcg_at_20": 0.51723, "ndcg_at_100": 0.52406, "ndcg_at_1000": 0.52406, "map_at_1": 0.27002, "map_at_3": 0.36313, "map_at_5": 0.38858, "map_at_10": 0.41445, "map_at_20": 0.42887, "map_at_100": 0.43287, "map_at_1000": 0.43287, "recall_at_1": 0.27002, "recall_at_3": 0.4083, "recall_at_5": 0.45639, "recall_at_10": 0.51155, "recall_at_20": 0.56073, "recall_at_100": 0.57824, "recall_at_1000": 0.57824, "accuracy": 0.27002, "precision_at_1": 0.52473, "precision_at_3": 0.32875, "precision_at_5": 0.25055, "precision_at_10": 0.16374, "precision_at_20": 0.10165, "precision_at_100": 0.02192, "precision_at_1000": 0.00219, "mrr_at_1": 0.524725, "mrr_at_3": 0.594322, "mrr_at_5": 0.603114, "mrr_at_10": 0.608168, "mrr_at_20": 0.609956, "mrr_at_100": 0.61008, "mrr_at_1000": 0.61008, "nauc_ndcg_at_1_max": 0.467384, "nauc_ndcg_at_1_std": 0.121971, "nauc_ndcg_at_1_diff1": 0.563775, "nauc_ndcg_at_3_max": 0.495369, "nauc_ndcg_at_3_std": 0.168181, "nauc_ndcg_at_3_diff1": 0.448822, "nauc_ndcg_at_5_max": 0.503916, "nauc_ndcg_at_5_std": 0.195613, "nauc_ndcg_at_5_diff1": 0.420927, "nauc_ndcg_at_10_max": 0.514072, "nauc_ndcg_at_10_std": 0.209622, "nauc_ndcg_at_10_diff1": 0.405445, "nauc_ndcg_at_20_max": 0.523419, "nauc_ndcg_at_20_std": 0.229286, "nauc_ndcg_at_20_diff1": 0.39976, "nauc_ndcg_at_100_max": 0.528427, "nauc_ndcg_at_100_std": 0.236012, "nauc_ndcg_at_100_diff1": 0.391751, "nauc_ndcg_at_1000_max": 0.528427, "nauc_ndcg_at_1000_std": 0.236012, "nauc_ndcg_at_1000_diff1": 0.391751, "nauc_map_at_1_max": 0.346779, "nauc_map_at_1_std": -0.126956, "nauc_map_at_1_diff1": 0.59467, "nauc_map_at_3_max": 0.417759, "nauc_map_at_3_std": -0.019695, "nauc_map_at_3_diff1": 0.478695, "nauc_map_at_5_max": 0.429903, "nauc_map_at_5_std": 0.03041, "nauc_map_at_5_diff1": 0.451061, "nauc_map_at_10_max": 0.443102, "nauc_map_at_10_std": 0.086103, "nauc_map_at_10_diff1": 0.422457, "nauc_map_at_20_max": 0.453684, "nauc_map_at_20_std": 0.116385, "nauc_map_at_20_diff1": 0.412549, "nauc_map_at_100_max": 0.457265, "nauc_map_at_100_std": 0.121391, "nauc_map_at_100_diff1": 0.408909, "nauc_map_at_1000_max": 0.457265, "nauc_map_at_1000_std": 0.121391, "nauc_map_at_1000_diff1": 0.408909, "nauc_recall_at_1_max": 0.346779, "nauc_recall_at_1_std": -0.126956, "nauc_recall_at_1_diff1": 0.59467, "nauc_recall_at_3_max": 0.443606, "nauc_recall_at_3_std": 0.033052, "nauc_recall_at_3_diff1": 0.408279, "nauc_recall_at_5_max": 0.460692, "nauc_recall_at_5_std": 0.118682, "nauc_recall_at_5_diff1": 0.352001, "nauc_recall_at_10_max": 0.485424, "nauc_recall_at_10_std": 0.209838, "nauc_recall_at_10_diff1": 0.303629, "nauc_recall_at_20_max": 0.51704, "nauc_recall_at_20_std": 0.296253, "nauc_recall_at_20_diff1": 0.272079, "nauc_recall_at_100_max": 0.526211, "nauc_recall_at_100_std": 0.314964, "nauc_recall_at_100_diff1": 0.249049, "nauc_recall_at_1000_max": 0.526211, "nauc_recall_at_1000_std": 0.314964, "nauc_recall_at_1000_diff1": 0.249049, "nauc_precision_at_1_max": 0.470753, "nauc_precision_at_1_std": 0.130719, "nauc_precision_at_1_diff1": 0.575359, "nauc_precision_at_3_max": 0.338673, "nauc_precision_at_3_std": 0.354053, "nauc_precision_at_3_diff1": 0.079862, "nauc_precision_at_5_max": 0.26229, "nauc_precision_at_5_std": 0.436448, "nauc_precision_at_5_diff1": -0.049704, "nauc_precision_at_10_max": 0.212496, "nauc_precision_at_10_std": 0.478937, "nauc_precision_at_10_diff1": -0.13245, "nauc_precision_at_20_max": 0.196913, "nauc_precision_at_20_std": 0.499352, "nauc_precision_at_20_diff1": -0.174579, "nauc_precision_at_100_max": 0.187412, "nauc_precision_at_100_std": 0.470433, "nauc_precision_at_100_diff1": -0.194154, "nauc_precision_at_1000_max": 0.187412, "nauc_precision_at_1000_std": 0.470433, "nauc_precision_at_1000_diff1": -0.194154, "nauc_mrr_at_1_max": 0.470753, "nauc_mrr_at_1_std": 0.130719, "nauc_mrr_at_1_diff1": 0.575359, "nauc_mrr_at_3_max": 0.539987, "nauc_mrr_at_3_std": 0.225344, "nauc_mrr_at_3_diff1": 0.519076, "nauc_mrr_at_5_max": 0.543806, "nauc_mrr_at_5_std": 0.243623, "nauc_mrr_at_5_diff1": 0.513948, "nauc_mrr_at_10_max": 0.548672, "nauc_mrr_at_10_std": 0.24911, "nauc_mrr_at_10_diff1": 0.513259, "nauc_mrr_at_20_max": 0.550306, "nauc_mrr_at_20_std": 0.250386, "nauc_mrr_at_20_diff1": 0.511987, "nauc_mrr_at_100_max": 0.550171, "nauc_mrr_at_100_std": 0.250533, "nauc_mrr_at_100_diff1": 0.511725, "nauc_mrr_at_1000_max": 0.550171, "nauc_mrr_at_1000_std": 0.250533, "nauc_mrr_at_1000_diff1": 0.511725, "hit_rate_at_1": 0.52473, "hit_rate_at_3": 0.68132, "hit_rate_at_5": 0.71978, "hit_rate_at_10": 0.75824, "hit_rate_at_20": 0.78297, "hit_rate_at_100": 0.78571, "hit_rate_at_1000": 0.78571, "main_score": 0.5029, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.51786, "ndcg_at_3": 0.5043, "ndcg_at_5": 0.49793, "ndcg_at_10": 0.50409, "ndcg_at_20": 0.51472, "ndcg_at_100": 0.52315, "ndcg_at_1000": 0.52315, "map_at_1": 0.27184, "map_at_3": 0.3666, "map_at_5": 0.38826, "map_at_10": 0.416, "map_at_20": 0.42916, "map_at_100": 0.43439, "map_at_1000": 0.43439, "recall_at_1": 0.27184, "recall_at_3": 0.4047, "recall_at_5": 0.44381, "recall_at_10": 0.50747, "recall_at_20": 0.54945, "recall_at_100": 0.57037, "recall_at_1000": 0.57037, "accuracy": 0.27184, "precision_at_1": 0.53846, "precision_at_3": 0.33242, "precision_at_5": 0.24615, "precision_at_10": 0.16374, "precision_at_20": 0.09766, "precision_at_100": 0.02151, "precision_at_1000": 0.00215, "mrr_at_1": 0.538462, "mrr_at_3": 0.602106, "mrr_at_5": 0.609799, "mrr_at_10": 0.613929, "mrr_at_20": 0.615307, "mrr_at_100": 0.615307, "mrr_at_1000": 0.615307, "nauc_ndcg_at_1_max": 0.531802, "nauc_ndcg_at_1_std": 0.189844, "nauc_ndcg_at_1_diff1": 0.534216, "nauc_ndcg_at_3_max": 0.574505, "nauc_ndcg_at_3_std": 0.241369, "nauc_ndcg_at_3_diff1": 0.439332, "nauc_ndcg_at_5_max": 0.598028, "nauc_ndcg_at_5_std": 0.280851, "nauc_ndcg_at_5_diff1": 0.420627, "nauc_ndcg_at_10_max": 0.607897, "nauc_ndcg_at_10_std": 0.295877, "nauc_ndcg_at_10_diff1": 0.416904, "nauc_ndcg_at_20_max": 0.611, "nauc_ndcg_at_20_std": 0.308637, "nauc_ndcg_at_20_diff1": 0.407427, "nauc_ndcg_at_100_max": 0.617989, "nauc_ndcg_at_100_std": 0.32295, "nauc_ndcg_at_100_diff1": 0.39928, "nauc_ndcg_at_1000_max": 0.617989, "nauc_ndcg_at_1000_std": 0.32295, "nauc_ndcg_at_1000_diff1": 0.39928, "nauc_map_at_1_max": 0.431524, "nauc_map_at_1_std": -0.004557, "nauc_map_at_1_diff1": 0.565307, "nauc_map_at_3_max": 0.49651, "nauc_map_at_3_std": 0.101421, "nauc_map_at_3_diff1": 0.445433, "nauc_map_at_5_max": 0.514866, "nauc_map_at_5_std": 0.148544, "nauc_map_at_5_diff1": 0.426678, "nauc_map_at_10_max": 0.523941, "nauc_map_at_10_std": 0.190539, "nauc_map_at_10_diff1": 0.403596, "nauc_map_at_20_max": 0.532912, "nauc_map_at_20_std": 0.212361, "nauc_map_at_20_diff1": 0.392991, "nauc_map_at_100_max": 0.537041, "nauc_map_at_100_std": 0.222454, "nauc_map_at_100_diff1": 0.388092, "nauc_map_at_1000_max": 0.537041, "nauc_map_at_1000_std": 0.222454, "nauc_map_at_1000_diff1": 0.388092, "nauc_recall_at_1_max": 0.431524, "nauc_recall_at_1_std": -0.004557, "nauc_recall_at_1_diff1": 0.565307, "nauc_recall_at_3_max": 0.52718, "nauc_recall_at_3_std": 0.152113, "nauc_recall_at_3_diff1": 0.393542, "nauc_recall_at_5_max": 0.561968, "nauc_recall_at_5_std": 0.243104, "nauc_recall_at_5_diff1": 0.361669, "nauc_recall_at_10_max": 0.579054, "nauc_recall_at_10_std": 0.325933, "nauc_recall_at_10_diff1": 0.323215, "nauc_recall_at_20_max": 0.607811, "nauc_recall_at_20_std": 0.391408, "nauc_recall_at_20_diff1": 0.285438, "nauc_recall_at_100_max": 0.622962, "nauc_recall_at_100_std": 0.426576, "nauc_recall_at_100_diff1": 0.264774, "nauc_recall_at_1000_max": 0.622962, "nauc_recall_at_1000_std": 0.426576, "nauc_recall_at_1000_diff1": 0.264774, "nauc_precision_at_1_max": 0.541286, "nauc_precision_at_1_std": 0.211063, "nauc_precision_at_1_diff1": 0.5318, "nauc_precision_at_3_max": 0.375602, "nauc_precision_at_3_std": 0.352298, "nauc_precision_at_3_diff1": 0.067273, "nauc_precision_at_5_max": 0.327173, "nauc_precision_at_5_std": 0.437917, "nauc_precision_at_5_diff1": -0.035704, "nauc_precision_at_10_max": 0.249853, "nauc_precision_at_10_std": 0.442546, "nauc_precision_at_10_diff1": -0.10061, "nauc_precision_at_20_max": 0.228478, "nauc_precision_at_20_std": 0.450182, "nauc_precision_at_20_diff1": -0.141893, "nauc_precision_at_100_max": 0.211331, "nauc_precision_at_100_std": 0.437784, "nauc_precision_at_100_diff1": -0.16182, "nauc_precision_at_1000_max": 0.211331, "nauc_precision_at_1000_std": 0.437784, "nauc_precision_at_1000_diff1": -0.16182, "nauc_mrr_at_1_max": 0.541286, "nauc_mrr_at_1_std": 0.211063, "nauc_mrr_at_1_diff1": 0.5318, "nauc_mrr_at_3_max": 0.6161, "nauc_mrr_at_3_std": 0.294385, "nauc_mrr_at_3_diff1": 0.503929, "nauc_mrr_at_5_max": 0.628535, "nauc_mrr_at_5_std": 0.312814, "nauc_mrr_at_5_diff1": 0.501554, "nauc_mrr_at_10_max": 0.633166, "nauc_mrr_at_10_std": 0.317854, "nauc_mrr_at_10_diff1": 0.503367, "nauc_mrr_at_20_max": 0.633418, "nauc_mrr_at_20_std": 0.320699, "nauc_mrr_at_20_diff1": 0.501743, "nauc_mrr_at_100_max": 0.633418, "nauc_mrr_at_100_std": 0.320699, "nauc_mrr_at_100_diff1": 0.501743, "nauc_mrr_at_1000_max": 0.633418, "nauc_mrr_at_1000_std": 0.320699, "nauc_mrr_at_1000_diff1": 0.501743, "hit_rate_at_1": 0.53846, "hit_rate_at_3": 0.68132, "hit_rate_at_5": 0.71429, "hit_rate_at_10": 0.74725, "hit_rate_at_20": 0.76648, "hit_rate_at_100": 0.76648, "hit_rate_at_1000": 0.76648, "main_score": 0.50409, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.64148, "ndcg_at_3": 0.64183, "ndcg_at_5": 0.64251, "ndcg_at_10": 0.66027, "ndcg_at_20": 0.68336, "ndcg_at_100": 0.70148, "ndcg_at_1000": 0.70148, "map_at_1": 0.3456, "map_at_3": 0.46805, "map_at_5": 0.50579, "map_at_10": 0.55018, "map_at_20": 0.57519, "map_at_100": 0.58613, "map_at_1000": 0.58613, "recall_at_1": 0.3456, "recall_at_3": 0.5175, "recall_at_5": 0.58937, "recall_at_10": 0.68626, "recall_at_20": 0.75762, "recall_at_100": 0.80425, "recall_at_1000": 0.80425, "accuracy": 0.3456, "precision_at_1": 0.66484, "precision_at_3": 0.42766, "precision_at_5": 0.33187, "precision_at_10": 0.22857, "precision_at_20": 0.14368, "precision_at_100": 0.0331, "precision_at_1000": 0.00331, "mrr_at_1": 0.664835, "mrr_at_3": 0.746795, "mrr_at_5": 0.757509, "mrr_at_10": 0.764928, "mrr_at_20": 0.765353, "mrr_at_100": 0.765697, "mrr_at_1000": 0.765697, "nauc_ndcg_at_1_max": 0.33142, "nauc_ndcg_at_1_std": -0.134939, "nauc_ndcg_at_1_diff1": 0.569712, "nauc_ndcg_at_3_max": 0.394624, "nauc_ndcg_at_3_std": -0.093101, "nauc_ndcg_at_3_diff1": 0.481546, "nauc_ndcg_at_5_max": 0.409444, "nauc_ndcg_at_5_std": -0.094196, "nauc_ndcg_at_5_diff1": 0.497379, "nauc_ndcg_at_10_max": 0.394126, "nauc_ndcg_at_10_std": -0.099346, "nauc_ndcg_at_10_diff1": 0.499717, "nauc_ndcg_at_20_max": 0.389147, "nauc_ndcg_at_20_std": -0.090597, "nauc_ndcg_at_20_diff1": 0.4921, "nauc_ndcg_at_100_max": 0.401029, "nauc_ndcg_at_100_std": -0.087549, "nauc_ndcg_at_100_diff1": 0.483695, "nauc_ndcg_at_1000_max": 0.401029, "nauc_ndcg_at_1000_std": -0.087549, "nauc_ndcg_at_1000_diff1": 0.483695, "nauc_map_at_1_max": 0.281652, "nauc_map_at_1_std": -0.229079, "nauc_map_at_1_diff1": 0.585293, "nauc_map_at_3_max": 0.347815, "nauc_map_at_3_std": -0.194834, "nauc_map_at_3_diff1": 0.531742, "nauc_map_at_5_max": 0.35956, "nauc_map_at_5_std": -0.167924, "nauc_map_at_5_diff1": 0.524066, "nauc_map_at_10_max": 0.358026, "nauc_map_at_10_std": -0.131222, "nauc_map_at_10_diff1": 0.50465, "nauc_map_at_20_max": 0.358818, "nauc_map_at_20_std": -0.111663, "nauc_map_at_20_diff1": 0.495076, "nauc_map_at_100_max": 0.362828, "nauc_map_at_100_std": -0.108448, "nauc_map_at_100_diff1": 0.491045, "nauc_map_at_1000_max": 0.362828, "nauc_map_at_1000_std": -0.108448, "nauc_map_at_1000_diff1": 0.491045, "nauc_recall_at_1_max": 0.281652, "nauc_recall_at_1_std": -0.229079, "nauc_recall_at_1_diff1": 0.585293, "nauc_recall_at_3_max": 0.363993, "nauc_recall_at_3_std": -0.178949, "nauc_recall_at_3_diff1": 0.484473, "nauc_recall_at_5_max": 0.381081, "nauc_recall_at_5_std": -0.134931, "nauc_recall_at_5_diff1": 0.473082, "nauc_recall_at_10_max": 0.346808, "nauc_recall_at_10_std": -0.08122, "nauc_recall_at_10_diff1": 0.424627, "nauc_recall_at_20_max": 0.345383, "nauc_recall_at_20_std": -0.035733, "nauc_recall_at_20_diff1": 0.387811, "nauc_recall_at_100_max": 0.384538, "nauc_recall_at_100_std": -0.025849, "nauc_recall_at_100_diff1": 0.357614, "nauc_recall_at_1000_max": 0.384538, "nauc_recall_at_1000_std": -0.025849, "nauc_recall_at_1000_diff1": 0.357614, "nauc_precision_at_1_max": 0.327884, "nauc_precision_at_1_std": -0.145348, "nauc_precision_at_1_diff1": 0.590371, "nauc_precision_at_3_max": 0.176085, "nauc_precision_at_3_std": 0.141553, "nauc_precision_at_3_diff1": -0.025664, "nauc_precision_at_5_max": 0.066589, "nauc_precision_at_5_std": 0.192894, "nauc_precision_at_5_diff1": -0.140235, "nauc_precision_at_10_max": -0.038308, "nauc_precision_at_10_std": 0.232318, "nauc_precision_at_10_diff1": -0.233665, "nauc_precision_at_20_max": -0.064465, "nauc_precision_at_20_std": 0.250241, "nauc_precision_at_20_diff1": -0.269467, "nauc_precision_at_100_max": -0.061706, "nauc_precision_at_100_std": 0.230844, "nauc_precision_at_100_diff1": -0.288185, "nauc_precision_at_1000_max": -0.061706, "nauc_precision_at_1000_std": 0.230844, "nauc_precision_at_1000_diff1": -0.288185, "nauc_mrr_at_1_max": 0.327884, "nauc_mrr_at_1_std": -0.145348, "nauc_mrr_at_1_diff1": 0.590371, "nauc_mrr_at_3_max": 0.394105, "nauc_mrr_at_3_std": -0.117944, "nauc_mrr_at_3_diff1": 0.573552, "nauc_mrr_at_5_max": 0.383359, "nauc_mrr_at_5_std": -0.1282, "nauc_mrr_at_5_diff1": 0.581578, "nauc_mrr_at_10_max": 0.374204, "nauc_mrr_at_10_std": -0.13577, "nauc_mrr_at_10_diff1": 0.575522, "nauc_mrr_at_20_max": 0.37617, "nauc_mrr_at_20_std": -0.13354, "nauc_mrr_at_20_diff1": 0.574632, "nauc_mrr_at_100_max": 0.375998, "nauc_mrr_at_100_std": -0.13226, "nauc_mrr_at_100_diff1": 0.574629, "nauc_mrr_at_1000_max": 0.375998, "nauc_mrr_at_1000_std": -0.13226, "nauc_mrr_at_1000_diff1": 0.574629, "hit_rate_at_1": 0.66484, "hit_rate_at_3": 0.8489, "hit_rate_at_5": 0.8956, "hit_rate_at_10": 0.95055, "hit_rate_at_20": 0.95604, "hit_rate_at_100": 0.96703, "hit_rate_at_1000": 0.96703, "main_score": 0.66027, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.5, "ndcg_at_3": 0.50341, "ndcg_at_5": 0.49735, "ndcg_at_10": 0.50496, "ndcg_at_20": 0.52431, "ndcg_at_100": 0.53527, "ndcg_at_1000": 0.53527, "map_at_1": 0.27643, "map_at_3": 0.37715, "map_at_5": 0.3972, "map_at_10": 0.4225, "map_at_20": 0.43992, "map_at_100": 0.44674, "map_at_1000": 0.44674, "recall_at_1": 0.27643, "recall_at_3": 0.4191, "recall_at_5": 0.45948, "recall_at_10": 0.51968, "recall_at_20": 0.5799, "recall_at_100": 0.60795, "recall_at_1000": 0.60795, "accuracy": 0.27643, "precision_at_1": 0.51923, "precision_at_3": 0.32784, "precision_at_5": 0.23901, "precision_at_10": 0.15549, "precision_at_20": 0.0989, "precision_at_100": 0.02255, "precision_at_1000": 0.00226, "mrr_at_1": 0.519231, "mrr_at_3": 0.589286, "mrr_at_5": 0.596841, "mrr_at_10": 0.602903, "mrr_at_20": 0.603939, "mrr_at_100": 0.604083, "mrr_at_1000": 0.604083, "nauc_ndcg_at_1_max": 0.46918, "nauc_ndcg_at_1_std": 0.164491, "nauc_ndcg_at_1_diff1": 0.495541, "nauc_ndcg_at_3_max": 0.523817, "nauc_ndcg_at_3_std": 0.240711, "nauc_ndcg_at_3_diff1": 0.420417, "nauc_ndcg_at_5_max": 0.536227, "nauc_ndcg_at_5_std": 0.274502, "nauc_ndcg_at_5_diff1": 0.413568, "nauc_ndcg_at_10_max": 0.553947, "nauc_ndcg_at_10_std": 0.298752, "nauc_ndcg_at_10_diff1": 0.419615, "nauc_ndcg_at_20_max": 0.557183, "nauc_ndcg_at_20_std": 0.331931, "nauc_ndcg_at_20_diff1": 0.404752, "nauc_ndcg_at_100_max": 0.570501, "nauc_ndcg_at_100_std": 0.341527, "nauc_ndcg_at_100_diff1": 0.399815, "nauc_ndcg_at_1000_max": 0.570501, "nauc_ndcg_at_1000_std": 0.341527, "nauc_ndcg_at_1000_diff1": 0.399815, "nauc_map_at_1_max": 0.412731, "nauc_map_at_1_std": 0.01053, "nauc_map_at_1_diff1": 0.561631, "nauc_map_at_3_max": 0.47888, "nauc_map_at_3_std": 0.121106, "nauc_map_at_3_diff1": 0.475048, "nauc_map_at_5_max": 0.485305, "nauc_map_at_5_std": 0.166463, "nauc_map_at_5_diff1": 0.456087, "nauc_map_at_10_max": 0.496922, "nauc_map_at_10_std": 0.21347, "nauc_map_at_10_diff1": 0.443643, "nauc_map_at_20_max": 0.499589, "nauc_map_at_20_std": 0.242701, "nauc_map_at_20_diff1": 0.428879, "nauc_map_at_100_max": 0.507136, "nauc_map_at_100_std": 0.25013, "nauc_map_at_100_diff1": 0.426152, "nauc_map_at_1000_max": 0.507136, "nauc_map_at_1000_std": 0.25013, "nauc_map_at_1000_diff1": 0.426152, "nauc_recall_at_1_max": 0.412731, "nauc_recall_at_1_std": 0.01053, "nauc_recall_at_1_diff1": 0.561631, "nauc_recall_at_3_max": 0.510077, "nauc_recall_at_3_std": 0.163166, "nauc_recall_at_3_diff1": 0.427795, "nauc_recall_at_5_max": 0.535414, "nauc_recall_at_5_std": 0.258904, "nauc_recall_at_5_diff1": 0.392405, "nauc_recall_at_10_max": 0.555284, "nauc_recall_at_10_std": 0.348319, "nauc_recall_at_10_diff1": 0.356139, "nauc_recall_at_20_max": 0.571479, "nauc_recall_at_20_std": 0.434899, "nauc_recall_at_20_diff1": 0.315098, "nauc_recall_at_100_max": 0.612486, "nauc_recall_at_100_std": 0.464791, "nauc_recall_at_100_diff1": 0.312764, "nauc_recall_at_1000_max": 0.612486, "nauc_recall_at_1000_std": 0.464791, "nauc_recall_at_1000_diff1": 0.312764, "nauc_precision_at_1_max": 0.461809, "nauc_precision_at_1_std": 0.165151, "nauc_precision_at_1_diff1": 0.502295, "nauc_precision_at_3_max": 0.323002, "nauc_precision_at_3_std": 0.339945, "nauc_precision_at_3_diff1": 0.083316, "nauc_precision_at_5_max": 0.242769, "nauc_precision_at_5_std": 0.411232, "nauc_precision_at_5_diff1": -0.018273, "nauc_precision_at_10_max": 0.181185, "nauc_precision_at_10_std": 0.43493, "nauc_precision_at_10_diff1": -0.089812, "nauc_precision_at_20_max": 0.134519, "nauc_precision_at_20_std": 0.423375, "nauc_precision_at_20_diff1": -0.154634, "nauc_precision_at_100_max": 0.143287, "nauc_precision_at_100_std": 0.377707, "nauc_precision_at_100_diff1": -0.160109, "nauc_precision_at_1000_max": 0.143287, "nauc_precision_at_1000_std": 0.377707, "nauc_precision_at_1000_diff1": -0.160109, "nauc_mrr_at_1_max": 0.461809, "nauc_mrr_at_1_std": 0.165151, "nauc_mrr_at_1_diff1": 0.502295, "nauc_mrr_at_3_max": 0.549949, "nauc_mrr_at_3_std": 0.269536, "nauc_mrr_at_3_diff1": 0.460576, "nauc_mrr_at_5_max": 0.559671, "nauc_mrr_at_5_std": 0.29617, "nauc_mrr_at_5_diff1": 0.455047, "nauc_mrr_at_10_max": 0.562358, "nauc_mrr_at_10_std": 0.300511, "nauc_mrr_at_10_diff1": 0.451853, "nauc_mrr_at_20_max": 0.563694, "nauc_mrr_at_20_std": 0.300418, "nauc_mrr_at_20_diff1": 0.45318, "nauc_mrr_at_100_max": 0.563941, "nauc_mrr_at_100_std": 0.300318, "nauc_mrr_at_100_diff1": 0.453695, "nauc_mrr_at_1000_max": 0.563941, "nauc_mrr_at_1000_std": 0.300318, "nauc_mrr_at_1000_diff1": 0.453695, "hit_rate_at_1": 0.51923, "hit_rate_at_3": 0.67308, "hit_rate_at_5": 0.70604, "hit_rate_at_10": 0.75275, "hit_rate_at_20": 0.76648, "hit_rate_at_100": 0.77198, "hit_rate_at_1000": 0.77198, "main_score": 0.50496, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.50962, "ndcg_at_3": 0.50306, "ndcg_at_5": 0.49731, "ndcg_at_10": 0.50123, "ndcg_at_20": 0.51314, "ndcg_at_100": 0.52185, "ndcg_at_1000": 0.52185, "map_at_1": 0.27821, "map_at_3": 0.36731, "map_at_5": 0.38938, "map_at_10": 0.41635, "map_at_20": 0.42952, "map_at_100": 0.43487, "map_at_1000": 0.43487, "recall_at_1": 0.27821, "recall_at_3": 0.40877, "recall_at_5": 0.44934, "recall_at_10": 0.508, "recall_at_20": 0.55539, "recall_at_100": 0.57634, "recall_at_1000": 0.57634, "accuracy": 0.27821, "precision_at_1": 0.53846, "precision_at_3": 0.33059, "precision_at_5": 0.24725, "precision_at_10": 0.16154, "precision_at_20": 0.09712, "precision_at_100": 0.02157, "precision_at_1000": 0.00216, "mrr_at_1": 0.538462, "mrr_at_3": 0.600733, "mrr_at_5": 0.607875, "mrr_at_10": 0.611081, "mrr_at_20": 0.612888, "mrr_at_100": 0.613041, "mrr_at_1000": 0.613041, "nauc_ndcg_at_1_max": 0.536286, "nauc_ndcg_at_1_std": 0.19088, "nauc_ndcg_at_1_diff1": 0.464412, "nauc_ndcg_at_3_max": 0.541979, "nauc_ndcg_at_3_std": 0.22911, "nauc_ndcg_at_3_diff1": 0.403393, "nauc_ndcg_at_5_max": 0.557064, "nauc_ndcg_at_5_std": 0.240955, "nauc_ndcg_at_5_diff1": 0.393841, "nauc_ndcg_at_10_max": 0.559623, "nauc_ndcg_at_10_std": 0.256392, "nauc_ndcg_at_10_diff1": 0.397758, "nauc_ndcg_at_20_max": 0.560059, "nauc_ndcg_at_20_std": 0.271129, "nauc_ndcg_at_20_diff1": 0.389108, "nauc_ndcg_at_100_max": 0.562764, "nauc_ndcg_at_100_std": 0.284388, "nauc_ndcg_at_100_diff1": 0.377616, "nauc_ndcg_at_1000_max": 0.562764, "nauc_ndcg_at_1000_std": 0.284388, "nauc_ndcg_at_1000_diff1": 0.377616, "nauc_map_at_1_max": 0.454965, "nauc_map_at_1_std": -0.03032, "nauc_map_at_1_diff1": 0.546372, "nauc_map_at_3_max": 0.491453, "nauc_map_at_3_std": 0.054118, "nauc_map_at_3_diff1": 0.461567, "nauc_map_at_5_max": 0.499765, "nauc_map_at_5_std": 0.096256, "nauc_map_at_5_diff1": 0.436595, "nauc_map_at_10_max": 0.500479, "nauc_map_at_10_std": 0.151214, "nauc_map_at_10_diff1": 0.41202, "nauc_map_at_20_max": 0.503823, "nauc_map_at_20_std": 0.17982, "nauc_map_at_20_diff1": 0.396015, "nauc_map_at_100_max": 0.504626, "nauc_map_at_100_std": 0.189963, "nauc_map_at_100_diff1": 0.388529, "nauc_map_at_1000_max": 0.504626, "nauc_map_at_1000_std": 0.189963, "nauc_map_at_1000_diff1": 0.388529, "nauc_recall_at_1_max": 0.454965, "nauc_recall_at_1_std": -0.03032, "nauc_recall_at_1_diff1": 0.546372, "nauc_recall_at_3_max": 0.494084, "nauc_recall_at_3_std": 0.076832, "nauc_recall_at_3_diff1": 0.429748, "nauc_recall_at_5_max": 0.520455, "nauc_recall_at_5_std": 0.154864, "nauc_recall_at_5_diff1": 0.385057, "nauc_recall_at_10_max": 0.525099, "nauc_recall_at_10_std": 0.256315, "nauc_recall_at_10_diff1": 0.351596, "nauc_recall_at_20_max": 0.545292, "nauc_recall_at_20_std": 0.325386, "nauc_recall_at_20_diff1": 0.313216, "nauc_recall_at_100_max": 0.550701, "nauc_recall_at_100_std": 0.357135, "nauc_recall_at_100_diff1": 0.286645, "nauc_recall_at_1000_max": 0.550701, "nauc_recall_at_1000_std": 0.357135, "nauc_recall_at_1000_diff1": 0.286645, "nauc_precision_at_1_max": 0.544438, "nauc_precision_at_1_std": 0.22237, "nauc_precision_at_1_diff1": 0.44939, "nauc_precision_at_3_max": 0.320749, "nauc_precision_at_3_std": 0.399485, "nauc_precision_at_3_diff1": 0.021329, "nauc_precision_at_5_max": 0.236478, "nauc_precision_at_5_std": 0.454661, "nauc_precision_at_5_diff1": -0.096263, "nauc_precision_at_10_max": 0.151131, "nauc_precision_at_10_std": 0.489204, "nauc_precision_at_10_diff1": -0.153625, "nauc_precision_at_20_max": 0.129428, "nauc_precision_at_20_std": 0.505703, "nauc_precision_at_20_diff1": -0.200282, "nauc_precision_at_100_max": 0.113332, "nauc_precision_at_100_std": 0.48803, "nauc_precision_at_100_diff1": -0.224769, "nauc_precision_at_1000_max": 0.113332, "nauc_precision_at_1000_std": 0.48803, "nauc_precision_at_1000_diff1": -0.224769, "nauc_mrr_at_1_max": 0.544438, "nauc_mrr_at_1_std": 0.22237, "nauc_mrr_at_1_diff1": 0.44939, "nauc_mrr_at_3_max": 0.595385, "nauc_mrr_at_3_std": 0.307183, "nauc_mrr_at_3_diff1": 0.423642, "nauc_mrr_at_5_max": 0.600503, "nauc_mrr_at_5_std": 0.320588, "nauc_mrr_at_5_diff1": 0.418737, "nauc_mrr_at_10_max": 0.602278, "nauc_mrr_at_10_std": 0.327394, "nauc_mrr_at_10_diff1": 0.418598, "nauc_mrr_at_20_max": 0.604479, "nauc_mrr_at_20_std": 0.326804, "nauc_mrr_at_20_diff1": 0.418698, "nauc_mrr_at_100_max": 0.604278, "nauc_mrr_at_100_std": 0.326414, "nauc_mrr_at_100_diff1": 0.418688, "nauc_mrr_at_1000_max": 0.604278, "nauc_mrr_at_1000_std": 0.326414, "nauc_mrr_at_1000_diff1": 0.418688, "hit_rate_at_1": 0.53846, "hit_rate_at_3": 0.68132, "hit_rate_at_5": 0.71154, "hit_rate_at_10": 0.73352, "hit_rate_at_20": 0.75824, "hit_rate_at_100": 0.76374, "hit_rate_at_1000": 0.76374, "main_score": 0.50123, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.24.9" }, { "ndcg_at_1": 0.50275, "ndcg_at_3": 0.48861, "ndcg_at_5": 0.48314, "ndcg_at_10": 0.48792, "ndcg_at_20": 0.5003, "ndcg_at_100": 0.50753, "ndcg_at_1000": 0.50753, "map_at_1": 0.26569, "map_at_3": 0.35681, "map_at_5": 0.37811, "map_at_10": 0.40358, "map_at_20": 0.4165, "map_at_100": 0.42123, "map_at_1000": 0.42123, "recall_at_1": 0.26569, "recall_at_3": 0.3948, "recall_at_5": 0.43529, "recall_at_10": 0.49133, "recall_at_20": 0.53981, "recall_at_100": 0.55782, "recall_at_1000": 0.55782, "accuracy": 0.26569, "precision_at_1": 0.51923, "precision_at_3": 0.32143, "precision_at_5": 0.23681, "precision_at_10": 0.15522, "precision_at_20": 0.09354, "precision_at_100": 0.02055, "precision_at_1000": 0.00205, "mrr_at_1": 0.519231, "mrr_at_3": 0.579212, "mrr_at_5": 0.587042, "mrr_at_10": 0.591732, "mrr_at_20": 0.592692, "mrr_at_100": 0.592783, "mrr_at_1000": 0.592783, "nauc_ndcg_at_1_max": 0.546337, "nauc_ndcg_at_1_std": 0.251325, "nauc_ndcg_at_1_diff1": 0.527924, "nauc_ndcg_at_3_max": 0.556191, "nauc_ndcg_at_3_std": 0.309587, "nauc_ndcg_at_3_diff1": 0.433114, "nauc_ndcg_at_5_max": 0.573446, "nauc_ndcg_at_5_std": 0.339622, "nauc_ndcg_at_5_diff1": 0.420915, "nauc_ndcg_at_10_max": 0.585213, "nauc_ndcg_at_10_std": 0.350977, "nauc_ndcg_at_10_diff1": 0.413764, "nauc_ndcg_at_20_max": 0.594789, "nauc_ndcg_at_20_std": 0.359199, "nauc_ndcg_at_20_diff1": 0.409675, "nauc_ndcg_at_100_max": 0.600686, "nauc_ndcg_at_100_std": 0.371148, "nauc_ndcg_at_100_diff1": 0.404419, "nauc_ndcg_at_1000_max": 0.600686, "nauc_ndcg_at_1000_std": 0.371148, "nauc_ndcg_at_1000_diff1": 0.404419, "nauc_map_at_1_max": 0.399289, "nauc_map_at_1_std": 0.00303, "nauc_map_at_1_diff1": 0.571786, "nauc_map_at_3_max": 0.480304, "nauc_map_at_3_std": 0.119421, "nauc_map_at_3_diff1": 0.47984, "nauc_map_at_5_max": 0.499268, "nauc_map_at_5_std": 0.175151, "nauc_map_at_5_diff1": 0.460288, "nauc_map_at_10_max": 0.511636, "nauc_map_at_10_std": 0.230006, "nauc_map_at_10_diff1": 0.434942, "nauc_map_at_20_max": 0.522426, "nauc_map_at_20_std": 0.25664, "nauc_map_at_20_diff1": 0.423011, "nauc_map_at_100_max": 0.526306, "nauc_map_at_100_std": 0.266941, "nauc_map_at_100_diff1": 0.419375, "nauc_map_at_1000_max": 0.526306, "nauc_map_at_1000_std": 0.266941, "nauc_map_at_1000_diff1": 0.419375, "nauc_recall_at_1_max": 0.399289, "nauc_recall_at_1_std": 0.00303, "nauc_recall_at_1_diff1": 0.571786, "nauc_recall_at_3_max": 0.497314, "nauc_recall_at_3_std": 0.153105, "nauc_recall_at_3_diff1": 0.432498, "nauc_recall_at_5_max": 0.537989, "nauc_recall_at_5_std": 0.256753, "nauc_recall_at_5_diff1": 0.395345, "nauc_recall_at_10_max": 0.562936, "nauc_recall_at_10_std": 0.353694, "nauc_recall_at_10_diff1": 0.343867, "nauc_recall_at_20_max": 0.60244, "nauc_recall_at_20_std": 0.418396, "nauc_recall_at_20_diff1": 0.310602, "nauc_recall_at_100_max": 0.619286, "nauc_recall_at_100_std": 0.448262, "nauc_recall_at_100_diff1": 0.300201, "nauc_recall_at_1000_max": 0.619286, "nauc_recall_at_1000_std": 0.448262, "nauc_recall_at_1000_diff1": 0.300201, "nauc_precision_at_1_max": 0.561846, "nauc_precision_at_1_std": 0.268168, "nauc_precision_at_1_diff1": 0.532305, "nauc_precision_at_3_max": 0.403738, "nauc_precision_at_3_std": 0.454858, "nauc_precision_at_3_diff1": 0.09551, "nauc_precision_at_5_max": 0.340815, "nauc_precision_at_5_std": 0.536617, "nauc_precision_at_5_diff1": -0.011411, "nauc_precision_at_10_max": 0.271944, "nauc_precision_at_10_std": 0.569133, "nauc_precision_at_10_diff1": -0.111031, "nauc_precision_at_20_max": 0.255443, "nauc_precision_at_20_std": 0.568168, "nauc_precision_at_20_diff1": -0.158215, "nauc_precision_at_100_max": 0.241751, "nauc_precision_at_100_std": 0.540796, "nauc_precision_at_100_diff1": -0.16917, "nauc_precision_at_1000_max": 0.241751, "nauc_precision_at_1000_std": 0.540796, "nauc_precision_at_1000_diff1": -0.16917, "nauc_mrr_at_1_max": 0.561846, "nauc_mrr_at_1_std": 0.268168, "nauc_mrr_at_1_diff1": 0.532305, "nauc_mrr_at_3_max": 0.604281, "nauc_mrr_at_3_std": 0.350065, "nauc_mrr_at_3_diff1": 0.498461, "nauc_mrr_at_5_max": 0.614959, "nauc_mrr_at_5_std": 0.372194, "nauc_mrr_at_5_diff1": 0.493989, "nauc_mrr_at_10_max": 0.623921, "nauc_mrr_at_10_std": 0.378395, "nauc_mrr_at_10_diff1": 0.490731, "nauc_mrr_at_20_max": 0.623874, "nauc_mrr_at_20_std": 0.378022, "nauc_mrr_at_20_diff1": 0.490242, "nauc_mrr_at_100_max": 0.623794, "nauc_mrr_at_100_std": 0.378066, "nauc_mrr_at_100_diff1": 0.490241, "nauc_mrr_at_1000_max": 0.623794, "nauc_mrr_at_1000_std": 0.378066, "nauc_mrr_at_1000_diff1": 0.490241, "hit_rate_at_1": 0.51923, "hit_rate_at_3": 0.65385, "hit_rate_at_5": 0.68681, "hit_rate_at_10": 0.72253, "hit_rate_at_20": 0.73626, "hit_rate_at_100": 0.73901, "hit_rate_at_1000": 0.73901, "main_score": 0.48792, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.24.9" } ] }, "evaluation_time": 5525.193988649029, "kg_co2_emissions": null, "date": 1791866490.58148, "evaluation_phases": [ { "name": "Data loading", "start": 3.975990694016218e-06, "end": 31.87538104399573, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 31.875413993984694, "end": 31.8754174989881, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 31.881993256974965, "end": 943.6925007919781, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 943.6925539809745, "end": 943.7400153929775, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 943.7530680929776, "end": 1860.1865177699947, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 1860.1865585409978, "end": 1860.2343216149893, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 1860.247305559984, "end": 2756.9094537279743, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 2756.9094852849958, "end": 2756.955766140978, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 2756.968997049, "end": 3656.883512388973, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 3656.8835426439764, "end": 3656.9287877219904, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 3656.9420265749795, "end": 4576.372573723987, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 4576.372625961987, "end": 4576.423452781979, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 4576.43822886399, "end": 5525.130116726999, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 5525.130153471982, "end": 5525.1822478259855, "split": "test", "subset": "portuguese" } ] }