{ "dataset_revision": "6d20dfc6799e48a4b1d9dab909980c06b9caa3d3", "task_name": "Vidore3PharmaceuticalsBM25sReranking.v2", "mteb_version": "2.24.7", "scores": { "test": [ { "ndcg_at_1": 0.4739, "ndcg_at_3": 0.47681, "ndcg_at_5": 0.47654, "ndcg_at_10": 0.48779, "ndcg_at_20": 0.50028, "ndcg_at_100": 0.50999, "ndcg_at_1000": 0.50999, "map_at_1": 0.25815, "map_at_3": 0.34736, "map_at_5": 0.37039, "map_at_10": 0.39787, "map_at_20": 0.41117, "map_at_100": 0.41661, "map_at_1000": 0.41661, "recall_at_1": 0.25815, "recall_at_3": 0.39156, "recall_at_5": 0.44125, "recall_at_10": 0.50974, "recall_at_20": 0.5516, "recall_at_100": 0.57824, "recall_at_1000": 0.57824, "accuracy": 0.25815, "precision_at_1": 0.5, "precision_at_3": 0.31685, "precision_at_5": 0.24066, "precision_at_10": 0.16099, "precision_at_20": 0.09863, "precision_at_100": 0.02192, "precision_at_1000": 0.00219, "mrr_at_1": 0.5, "mrr_at_3": 0.571886, "mrr_at_5": 0.582601, "mrr_at_10": 0.589879, "mrr_at_20": 0.591069, "mrr_at_100": 0.591198, "mrr_at_1000": 0.591198, "nauc_ndcg_at_1_max": 0.468133, "nauc_ndcg_at_1_std": 0.30005, "nauc_ndcg_at_1_diff1": 0.498388, "nauc_ndcg_at_3_max": 0.520336, "nauc_ndcg_at_3_std": 0.357849, "nauc_ndcg_at_3_diff1": 0.46379, "nauc_ndcg_at_5_max": 0.536419, "nauc_ndcg_at_5_std": 0.374321, "nauc_ndcg_at_5_diff1": 0.461305, "nauc_ndcg_at_10_max": 0.552869, "nauc_ndcg_at_10_std": 0.401583, "nauc_ndcg_at_10_diff1": 0.467897, "nauc_ndcg_at_20_max": 0.561207, "nauc_ndcg_at_20_std": 0.416398, "nauc_ndcg_at_20_diff1": 0.465812, "nauc_ndcg_at_100_max": 0.571487, "nauc_ndcg_at_100_std": 0.431305, "nauc_ndcg_at_100_diff1": 0.46183, "nauc_ndcg_at_1000_max": 0.571487, "nauc_ndcg_at_1000_std": 0.431305, "nauc_ndcg_at_1000_diff1": 0.46183, "nauc_map_at_1_max": 0.339056, "nauc_map_at_1_std": 0.117109, "nauc_map_at_1_diff1": 0.49308, "nauc_map_at_3_max": 0.419386, "nauc_map_at_3_std": 0.218934, "nauc_map_at_3_diff1": 0.445817, "nauc_map_at_5_max": 0.444554, "nauc_map_at_5_std": 0.256297, "nauc_map_at_5_diff1": 0.442533, "nauc_map_at_10_max": 0.470196, "nauc_map_at_10_std": 0.30043, "nauc_map_at_10_diff1": 0.438465, "nauc_map_at_20_max": 0.48217, "nauc_map_at_20_std": 0.322508, "nauc_map_at_20_diff1": 0.430899, "nauc_map_at_100_max": 0.488445, "nauc_map_at_100_std": 0.331878, "nauc_map_at_100_diff1": 0.428367, "nauc_map_at_1000_max": 0.488445, "nauc_map_at_1000_std": 0.331878, "nauc_map_at_1000_diff1": 0.428367, "nauc_recall_at_1_max": 0.339056, "nauc_recall_at_1_std": 0.117109, "nauc_recall_at_1_diff1": 0.49308, "nauc_recall_at_3_max": 0.42679, "nauc_recall_at_3_std": 0.240124, "nauc_recall_at_3_diff1": 0.430921, "nauc_recall_at_5_max": 0.457625, "nauc_recall_at_5_std": 0.294034, "nauc_recall_at_5_diff1": 0.418461, "nauc_recall_at_10_max": 0.501146, "nauc_recall_at_10_std": 0.381988, "nauc_recall_at_10_diff1": 0.411767, "nauc_recall_at_20_max": 0.532774, "nauc_recall_at_20_std": 0.44234, "nauc_recall_at_20_diff1": 0.38828, "nauc_recall_at_100_max": 0.561541, "nauc_recall_at_100_std": 0.482568, "nauc_recall_at_100_diff1": 0.381606, "nauc_recall_at_1000_max": 0.561541, "nauc_recall_at_1000_std": 0.482568, "nauc_recall_at_1000_diff1": 0.381606, "nauc_precision_at_1_max": 0.48785, "nauc_precision_at_1_std": 0.329153, "nauc_precision_at_1_diff1": 0.498298, "nauc_precision_at_3_max": 0.428742, "nauc_precision_at_3_std": 0.412402, "nauc_precision_at_3_diff1": 0.204472, "nauc_precision_at_5_max": 0.397981, "nauc_precision_at_5_std": 0.440314, "nauc_precision_at_5_diff1": 0.118078, "nauc_precision_at_10_max": 0.345369, "nauc_precision_at_10_std": 0.451762, "nauc_precision_at_10_diff1": 0.045971, "nauc_precision_at_20_max": 0.302147, "nauc_precision_at_20_std": 0.438592, "nauc_precision_at_20_diff1": -0.020305, "nauc_precision_at_100_max": 0.275973, "nauc_precision_at_100_std": 0.416473, "nauc_precision_at_100_diff1": -0.043161, "nauc_precision_at_1000_max": 0.275973, "nauc_precision_at_1000_std": 0.416473, "nauc_precision_at_1000_diff1": -0.043161, "nauc_mrr_at_1_max": 0.48785, "nauc_mrr_at_1_std": 0.329153, "nauc_mrr_at_1_diff1": 0.498298, "nauc_mrr_at_3_max": 0.561039, "nauc_mrr_at_3_std": 0.409104, "nauc_mrr_at_3_diff1": 0.537442, "nauc_mrr_at_5_max": 0.565362, "nauc_mrr_at_5_std": 0.414965, "nauc_mrr_at_5_diff1": 0.538639, "nauc_mrr_at_10_max": 0.573407, "nauc_mrr_at_10_std": 0.428805, "nauc_mrr_at_10_diff1": 0.538771, "nauc_mrr_at_20_max": 0.572323, "nauc_mrr_at_20_std": 0.427592, "nauc_mrr_at_20_diff1": 0.538556, "nauc_mrr_at_100_max": 0.572302, "nauc_mrr_at_100_std": 0.427674, "nauc_mrr_at_100_diff1": 0.538497, "nauc_mrr_at_1000_max": 0.572302, "nauc_mrr_at_1000_std": 0.427674, "nauc_mrr_at_1000_diff1": 0.538497, "hit_rate_at_1": 0.5, "hit_rate_at_3": 0.66209, "hit_rate_at_5": 0.70879, "hit_rate_at_10": 0.76374, "hit_rate_at_20": 0.78022, "hit_rate_at_100": 0.78571, "hit_rate_at_1000": 0.78571, "main_score": 0.48779, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.45604, "ndcg_at_3": 0.4696, "ndcg_at_5": 0.46567, "ndcg_at_10": 0.47415, "ndcg_at_20": 0.48769, "ndcg_at_100": 0.49691, "ndcg_at_1000": 0.49691, "map_at_1": 0.23461, "map_at_3": 0.33072, "map_at_5": 0.35478, "map_at_10": 0.38065, "map_at_20": 0.39564, "map_at_100": 0.40067, "map_at_1000": 0.40067, "recall_at_1": 0.23461, "recall_at_3": 0.38586, "recall_at_5": 0.43269, "recall_at_10": 0.49748, "recall_at_20": 0.54518, "recall_at_100": 0.57037, "recall_at_1000": 0.57037, "accuracy": 0.23461, "precision_at_1": 0.48077, "precision_at_3": 0.32143, "precision_at_5": 0.23901, "precision_at_10": 0.15879, "precision_at_20": 0.0978, "precision_at_100": 0.02151, "precision_at_1000": 0.00215, "mrr_at_1": 0.480769, "mrr_at_3": 0.569597, "mrr_at_5": 0.577015, "mrr_at_10": 0.582485, "mrr_at_20": 0.583548, "mrr_at_100": 0.583718, "mrr_at_1000": 0.583718, "nauc_ndcg_at_1_max": 0.48645, "nauc_ndcg_at_1_std": 0.2781, "nauc_ndcg_at_1_diff1": 0.491722, "nauc_ndcg_at_3_max": 0.526974, "nauc_ndcg_at_3_std": 0.376198, "nauc_ndcg_at_3_diff1": 0.432355, "nauc_ndcg_at_5_max": 0.555835, "nauc_ndcg_at_5_std": 0.406461, "nauc_ndcg_at_5_diff1": 0.422409, "nauc_ndcg_at_10_max": 0.580975, "nauc_ndcg_at_10_std": 0.44256, "nauc_ndcg_at_10_diff1": 0.424861, "nauc_ndcg_at_20_max": 0.590513, "nauc_ndcg_at_20_std": 0.46711, "nauc_ndcg_at_20_diff1": 0.421422, "nauc_ndcg_at_100_max": 0.590514, "nauc_ndcg_at_100_std": 0.477199, "nauc_ndcg_at_100_diff1": 0.420415, "nauc_ndcg_at_1000_max": 0.590514, "nauc_ndcg_at_1000_std": 0.477199, "nauc_ndcg_at_1000_diff1": 0.420415, "nauc_map_at_1_max": 0.383235, "nauc_map_at_1_std": 0.125118, "nauc_map_at_1_diff1": 0.548279, "nauc_map_at_3_max": 0.43732, "nauc_map_at_3_std": 0.239823, "nauc_map_at_3_diff1": 0.443903, "nauc_map_at_5_max": 0.472817, "nauc_map_at_5_std": 0.287918, "nauc_map_at_5_diff1": 0.427431, "nauc_map_at_10_max": 0.500004, "nauc_map_at_10_std": 0.340089, "nauc_map_at_10_diff1": 0.405747, "nauc_map_at_20_max": 0.510097, "nauc_map_at_20_std": 0.367568, "nauc_map_at_20_diff1": 0.39167, "nauc_map_at_100_max": 0.510406, "nauc_map_at_100_std": 0.374344, "nauc_map_at_100_diff1": 0.390063, "nauc_map_at_1000_max": 0.510406, "nauc_map_at_1000_std": 0.374344, "nauc_map_at_1000_diff1": 0.390063, "nauc_recall_at_1_max": 0.383235, "nauc_recall_at_1_std": 0.125118, "nauc_recall_at_1_diff1": 0.548279, "nauc_recall_at_3_max": 0.424434, "nauc_recall_at_3_std": 0.272532, "nauc_recall_at_3_diff1": 0.396136, "nauc_recall_at_5_max": 0.486764, "nauc_recall_at_5_std": 0.357017, "nauc_recall_at_5_diff1": 0.364075, "nauc_recall_at_10_max": 0.529681, "nauc_recall_at_10_std": 0.458095, "nauc_recall_at_10_diff1": 0.320197, "nauc_recall_at_20_max": 0.574209, "nauc_recall_at_20_std": 0.540855, "nauc_recall_at_20_diff1": 0.302085, "nauc_recall_at_100_max": 0.57336, "nauc_recall_at_100_std": 0.567301, "nauc_recall_at_100_diff1": 0.304437, "nauc_recall_at_1000_max": 0.57336, "nauc_recall_at_1000_std": 0.567301, "nauc_recall_at_1000_diff1": 0.304437, "nauc_precision_at_1_max": 0.496731, "nauc_precision_at_1_std": 0.295768, "nauc_precision_at_1_diff1": 0.493586, "nauc_precision_at_3_max": 0.42246, "nauc_precision_at_3_std": 0.458273, "nauc_precision_at_3_diff1": 0.101993, "nauc_precision_at_5_max": 0.405734, "nauc_precision_at_5_std": 0.492751, "nauc_precision_at_5_diff1": 0.010042, "nauc_precision_at_10_max": 0.350309, "nauc_precision_at_10_std": 0.499751, "nauc_precision_at_10_diff1": -0.0795, "nauc_precision_at_20_max": 0.303707, "nauc_precision_at_20_std": 0.488235, "nauc_precision_at_20_diff1": -0.124179, "nauc_precision_at_100_max": 0.261383, "nauc_precision_at_100_std": 0.460041, "nauc_precision_at_100_diff1": -0.124515, "nauc_precision_at_1000_max": 0.261383, "nauc_precision_at_1000_std": 0.460041, "nauc_precision_at_1000_diff1": -0.124515, "nauc_mrr_at_1_max": 0.496731, "nauc_mrr_at_1_std": 0.295768, "nauc_mrr_at_1_diff1": 0.493586, "nauc_mrr_at_3_max": 0.572179, "nauc_mrr_at_3_std": 0.417428, "nauc_mrr_at_3_diff1": 0.48351, "nauc_mrr_at_5_max": 0.588338, "nauc_mrr_at_5_std": 0.437484, "nauc_mrr_at_5_diff1": 0.477748, "nauc_mrr_at_10_max": 0.591599, "nauc_mrr_at_10_std": 0.442706, "nauc_mrr_at_10_diff1": 0.480765, "nauc_mrr_at_20_max": 0.59248, "nauc_mrr_at_20_std": 0.444283, "nauc_mrr_at_20_diff1": 0.480783, "nauc_mrr_at_100_max": 0.592403, "nauc_mrr_at_100_std": 0.444202, "nauc_mrr_at_100_diff1": 0.480669, "nauc_mrr_at_1000_max": 0.592403, "nauc_mrr_at_1000_std": 0.444202, "nauc_mrr_at_1000_diff1": 0.480669, "hit_rate_at_1": 0.48077, "hit_rate_at_3": 0.67582, "hit_rate_at_5": 0.70604, "hit_rate_at_10": 0.74725, "hit_rate_at_20": 0.76099, "hit_rate_at_100": 0.76648, "hit_rate_at_1000": 0.76648, "main_score": 0.47415, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.60714, "ndcg_at_3": 0.6235, "ndcg_at_5": 0.62601, "ndcg_at_10": 0.63785, "ndcg_at_20": 0.66364, "ndcg_at_100": 0.6854, "ndcg_at_1000": 0.6854, "map_at_1": 0.3228, "map_at_3": 0.44977, "map_at_5": 0.48852, "map_at_10": 0.5271, "map_at_20": 0.55334, "map_at_100": 0.56486, "map_at_1000": 0.56486, "recall_at_1": 0.3228, "recall_at_3": 0.50445, "recall_at_5": 0.57471, "recall_at_10": 0.66409, "recall_at_20": 0.74431, "recall_at_100": 0.80425, "recall_at_1000": 0.80425, "accuracy": 0.3228, "precision_at_1": 0.63187, "precision_at_3": 0.42674, "precision_at_5": 0.32967, "precision_at_10": 0.2206, "precision_at_20": 0.14093, "precision_at_100": 0.0331, "precision_at_1000": 0.00331, "mrr_at_1": 0.631868, "mrr_at_3": 0.723443, "mrr_at_5": 0.732372, "mrr_at_10": 0.740263, "mrr_at_20": 0.741978, "mrr_at_100": 0.742509, "mrr_at_1000": 0.742509, "nauc_ndcg_at_1_max": 0.351698, "nauc_ndcg_at_1_std": 0.013177, "nauc_ndcg_at_1_diff1": 0.388353, "nauc_ndcg_at_3_max": 0.389356, "nauc_ndcg_at_3_std": 0.07527, "nauc_ndcg_at_3_diff1": 0.324689, "nauc_ndcg_at_5_max": 0.404503, "nauc_ndcg_at_5_std": 0.06745, "nauc_ndcg_at_5_diff1": 0.341745, "nauc_ndcg_at_10_max": 0.397805, "nauc_ndcg_at_10_std": 0.059213, "nauc_ndcg_at_10_diff1": 0.348947, "nauc_ndcg_at_20_max": 0.417886, "nauc_ndcg_at_20_std": 0.09769, "nauc_ndcg_at_20_diff1": 0.329091, "nauc_ndcg_at_100_max": 0.436625, "nauc_ndcg_at_100_std": 0.137636, "nauc_ndcg_at_100_diff1": 0.310108, "nauc_ndcg_at_1000_max": 0.436625, "nauc_ndcg_at_1000_std": 0.137636, "nauc_ndcg_at_1000_diff1": 0.310108, "nauc_map_at_1_max": 0.238631, "nauc_map_at_1_std": -0.135354, "nauc_map_at_1_diff1": 0.430833, "nauc_map_at_3_max": 0.298242, "nauc_map_at_3_std": -0.082523, "nauc_map_at_3_diff1": 0.364611, "nauc_map_at_5_max": 0.315968, "nauc_map_at_5_std": -0.058075, "nauc_map_at_5_diff1": 0.363802, "nauc_map_at_10_max": 0.329556, "nauc_map_at_10_std": -0.020065, "nauc_map_at_10_diff1": 0.352999, "nauc_map_at_20_max": 0.35438, "nauc_map_at_20_std": 0.028128, "nauc_map_at_20_diff1": 0.334216, "nauc_map_at_100_max": 0.366027, "nauc_map_at_100_std": 0.048943, "nauc_map_at_100_diff1": 0.325971, "nauc_map_at_1000_max": 0.366027, "nauc_map_at_1000_std": 0.048943, "nauc_map_at_1000_diff1": 0.325971, "nauc_recall_at_1_max": 0.238631, "nauc_recall_at_1_std": -0.135354, "nauc_recall_at_1_diff1": 0.430833, "nauc_recall_at_3_max": 0.297609, "nauc_recall_at_3_std": -0.073115, "nauc_recall_at_3_diff1": 0.332089, "nauc_recall_at_5_max": 0.321264, "nauc_recall_at_5_std": -0.043272, "nauc_recall_at_5_diff1": 0.33673, "nauc_recall_at_10_max": 0.328048, "nauc_recall_at_10_std": 0.015428, "nauc_recall_at_10_diff1": 0.310958, "nauc_recall_at_20_max": 0.395945, "nauc_recall_at_20_std": 0.144828, "nauc_recall_at_20_diff1": 0.242433, "nauc_recall_at_100_max": 0.449567, "nauc_recall_at_100_std": 0.281656, "nauc_recall_at_100_diff1": 0.176195, "nauc_recall_at_1000_max": 0.449567, "nauc_recall_at_1000_std": 0.281656, "nauc_recall_at_1000_diff1": 0.176195, "nauc_precision_at_1_max": 0.360042, "nauc_precision_at_1_std": 0.047514, "nauc_precision_at_1_diff1": 0.380359, "nauc_precision_at_3_max": 0.227419, "nauc_precision_at_3_std": 0.273745, "nauc_precision_at_3_diff1": -0.066101, "nauc_precision_at_5_max": 0.183155, "nauc_precision_at_5_std": 0.297288, "nauc_precision_at_5_diff1": -0.10784, "nauc_precision_at_10_max": 0.10696, "nauc_precision_at_10_std": 0.306224, "nauc_precision_at_10_diff1": -0.157373, "nauc_precision_at_20_max": 0.106654, "nauc_precision_at_20_std": 0.362535, "nauc_precision_at_20_diff1": -0.218581, "nauc_precision_at_100_max": 0.092157, "nauc_precision_at_100_std": 0.378638, "nauc_precision_at_100_diff1": -0.254368, "nauc_precision_at_1000_max": 0.092157, "nauc_precision_at_1000_std": 0.378638, "nauc_precision_at_1000_diff1": -0.254368, "nauc_mrr_at_1_max": 0.360042, "nauc_mrr_at_1_std": 0.047514, "nauc_mrr_at_1_diff1": 0.380359, "nauc_mrr_at_3_max": 0.408686, "nauc_mrr_at_3_std": 0.097936, "nauc_mrr_at_3_diff1": 0.364638, "nauc_mrr_at_5_max": 0.417042, "nauc_mrr_at_5_std": 0.105966, "nauc_mrr_at_5_diff1": 0.365353, "nauc_mrr_at_10_max": 0.414185, "nauc_mrr_at_10_std": 0.105178, "nauc_mrr_at_10_diff1": 0.363389, "nauc_mrr_at_20_max": 0.412144, "nauc_mrr_at_20_std": 0.101761, "nauc_mrr_at_20_diff1": 0.364937, "nauc_mrr_at_100_max": 0.411134, "nauc_mrr_at_100_std": 0.100597, "nauc_mrr_at_100_diff1": 0.365531, "nauc_mrr_at_1000_max": 0.411134, "nauc_mrr_at_1000_std": 0.100597, "nauc_mrr_at_1000_diff1": 0.365531, "hit_rate_at_1": 0.63187, "hit_rate_at_3": 0.83242, "hit_rate_at_5": 0.87088, "hit_rate_at_10": 0.92857, "hit_rate_at_20": 0.95055, "hit_rate_at_100": 0.96703, "hit_rate_at_1000": 0.96703, "main_score": 0.63785, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.43956, "ndcg_at_3": 0.4656, "ndcg_at_5": 0.47145, "ndcg_at_10": 0.47861, "ndcg_at_20": 0.49663, "ndcg_at_100": 0.50973, "ndcg_at_1000": 0.50973, "map_at_1": 0.23809, "map_at_3": 0.34153, "map_at_5": 0.36908, "map_at_10": 0.39395, "map_at_20": 0.41043, "map_at_100": 0.41727, "map_at_1000": 0.41727, "recall_at_1": 0.23809, "recall_at_3": 0.39723, "recall_at_5": 0.45362, "recall_at_10": 0.51342, "recall_at_20": 0.57082, "recall_at_100": 0.60795, "recall_at_1000": 0.60795, "accuracy": 0.23809, "precision_at_1": 0.46154, "precision_at_3": 0.31502, "precision_at_5": 0.24231, "precision_at_10": 0.15852, "precision_at_20": 0.09904, "precision_at_100": 0.02255, "precision_at_1000": 0.00226, "mrr_at_1": 0.461538, "mrr_at_3": 0.548077, "mrr_at_5": 0.558379, "mrr_at_10": 0.563235, "mrr_at_20": 0.565209, "mrr_at_100": 0.565626, "mrr_at_1000": 0.565626, "nauc_ndcg_at_1_max": 0.431496, "nauc_ndcg_at_1_std": 0.26007, "nauc_ndcg_at_1_diff1": 0.511985, "nauc_ndcg_at_3_max": 0.510239, "nauc_ndcg_at_3_std": 0.353768, "nauc_ndcg_at_3_diff1": 0.502566, "nauc_ndcg_at_5_max": 0.534448, "nauc_ndcg_at_5_std": 0.375958, "nauc_ndcg_at_5_diff1": 0.505243, "nauc_ndcg_at_10_max": 0.54056, "nauc_ndcg_at_10_std": 0.389946, "nauc_ndcg_at_10_diff1": 0.503561, "nauc_ndcg_at_20_max": 0.554511, "nauc_ndcg_at_20_std": 0.418262, "nauc_ndcg_at_20_diff1": 0.499643, "nauc_ndcg_at_100_max": 0.567113, "nauc_ndcg_at_100_std": 0.441676, "nauc_ndcg_at_100_diff1": 0.492437, "nauc_ndcg_at_1000_max": 0.567113, "nauc_ndcg_at_1000_std": 0.441676, "nauc_ndcg_at_1000_diff1": 0.492437, "nauc_map_at_1_max": 0.376365, "nauc_map_at_1_std": 0.13136, "nauc_map_at_1_diff1": 0.543871, "nauc_map_at_3_max": 0.429658, "nauc_map_at_3_std": 0.213794, "nauc_map_at_3_diff1": 0.492835, "nauc_map_at_5_max": 0.455988, "nauc_map_at_5_std": 0.254749, "nauc_map_at_5_diff1": 0.489034, "nauc_map_at_10_max": 0.472959, "nauc_map_at_10_std": 0.293286, "nauc_map_at_10_diff1": 0.482794, "nauc_map_at_20_max": 0.485861, "nauc_map_at_20_std": 0.322966, "nauc_map_at_20_diff1": 0.477342, "nauc_map_at_100_max": 0.493728, "nauc_map_at_100_std": 0.337661, "nauc_map_at_100_diff1": 0.473644, "nauc_map_at_1000_max": 0.493728, "nauc_map_at_1000_std": 0.337661, "nauc_map_at_1000_diff1": 0.473644, "nauc_recall_at_1_max": 0.376365, "nauc_recall_at_1_std": 0.13136, "nauc_recall_at_1_diff1": 0.543871, "nauc_recall_at_3_max": 0.433193, "nauc_recall_at_3_std": 0.236902, "nauc_recall_at_3_diff1": 0.453366, "nauc_recall_at_5_max": 0.481612, "nauc_recall_at_5_std": 0.31092, "nauc_recall_at_5_diff1": 0.444605, "nauc_recall_at_10_max": 0.52059, "nauc_recall_at_10_std": 0.40125, "nauc_recall_at_10_diff1": 0.428342, "nauc_recall_at_20_max": 0.55971, "nauc_recall_at_20_std": 0.481973, "nauc_recall_at_20_diff1": 0.420282, "nauc_recall_at_100_max": 0.589085, "nauc_recall_at_100_std": 0.544437, "nauc_recall_at_100_diff1": 0.402886, "nauc_recall_at_1000_max": 0.589085, "nauc_recall_at_1000_std": 0.544437, "nauc_recall_at_1000_diff1": 0.402886, "nauc_precision_at_1_max": 0.451766, "nauc_precision_at_1_std": 0.299159, "nauc_precision_at_1_diff1": 0.49648, "nauc_precision_at_3_max": 0.420602, "nauc_precision_at_3_std": 0.426198, "nauc_precision_at_3_diff1": 0.236337, "nauc_precision_at_5_max": 0.394366, "nauc_precision_at_5_std": 0.45391, "nauc_precision_at_5_diff1": 0.16955, "nauc_precision_at_10_max": 0.311396, "nauc_precision_at_10_std": 0.43306, "nauc_precision_at_10_diff1": 0.093853, "nauc_precision_at_20_max": 0.279754, "nauc_precision_at_20_std": 0.440004, "nauc_precision_at_20_diff1": 0.028766, "nauc_precision_at_100_max": 0.278334, "nauc_precision_at_100_std": 0.44998, "nauc_precision_at_100_diff1": -0.001628, "nauc_precision_at_1000_max": 0.278334, "nauc_precision_at_1000_std": 0.44998, "nauc_precision_at_1000_diff1": -0.001628, "nauc_mrr_at_1_max": 0.451766, "nauc_mrr_at_1_std": 0.299159, "nauc_mrr_at_1_diff1": 0.49648, "nauc_mrr_at_3_max": 0.552879, "nauc_mrr_at_3_std": 0.416169, "nauc_mrr_at_3_diff1": 0.527876, "nauc_mrr_at_5_max": 0.565611, "nauc_mrr_at_5_std": 0.434109, "nauc_mrr_at_5_diff1": 0.528775, "nauc_mrr_at_10_max": 0.565572, "nauc_mrr_at_10_std": 0.436866, "nauc_mrr_at_10_diff1": 0.528074, "nauc_mrr_at_20_max": 0.565025, "nauc_mrr_at_20_std": 0.436957, "nauc_mrr_at_20_diff1": 0.528416, "nauc_mrr_at_100_max": 0.56486, "nauc_mrr_at_100_std": 0.436718, "nauc_mrr_at_100_diff1": 0.528951, "nauc_mrr_at_1000_max": 0.56486, "nauc_mrr_at_1000_std": 0.436718, "nauc_mrr_at_1000_diff1": 0.528951, "hit_rate_at_1": 0.46154, "hit_rate_at_3": 0.6511, "hit_rate_at_5": 0.69505, "hit_rate_at_10": 0.73352, "hit_rate_at_20": 0.76099, "hit_rate_at_100": 0.77198, "hit_rate_at_1000": 0.77198, "main_score": 0.47861, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.4533, "ndcg_at_3": 0.46918, "ndcg_at_5": 0.47351, "ndcg_at_10": 0.48045, "ndcg_at_20": 0.49134, "ndcg_at_100": 0.50082, "ndcg_at_1000": 0.50082, "map_at_1": 0.24136, "map_at_3": 0.33959, "map_at_5": 0.36573, "map_at_10": 0.39193, "map_at_20": 0.40623, "map_at_100": 0.41147, "map_at_1000": 0.41147, "recall_at_1": 0.24136, "recall_at_3": 0.38614, "recall_at_5": 0.44279, "recall_at_10": 0.50984, "recall_at_20": 0.54971, "recall_at_100": 0.57634, "recall_at_1000": 0.57634, "accuracy": 0.24136, "precision_at_1": 0.47802, "precision_at_3": 0.31319, "precision_at_5": 0.24451, "precision_at_10": 0.16126, "precision_at_20": 0.0978, "precision_at_100": 0.02157, "precision_at_1000": 0.00216, "mrr_at_1": 0.478022, "mrr_at_3": 0.553114, "mrr_at_5": 0.567949, "mrr_at_10": 0.57251, "mrr_at_20": 0.574072, "mrr_at_100": 0.57417, "mrr_at_1000": 0.57417, "nauc_ndcg_at_1_max": 0.453085, "nauc_ndcg_at_1_std": 0.274566, "nauc_ndcg_at_1_diff1": 0.461762, "nauc_ndcg_at_3_max": 0.506912, "nauc_ndcg_at_3_std": 0.339085, "nauc_ndcg_at_3_diff1": 0.434434, "nauc_ndcg_at_5_max": 0.542923, "nauc_ndcg_at_5_std": 0.383369, "nauc_ndcg_at_5_diff1": 0.436296, "nauc_ndcg_at_10_max": 0.563232, "nauc_ndcg_at_10_std": 0.407504, "nauc_ndcg_at_10_diff1": 0.455357, "nauc_ndcg_at_20_max": 0.564336, "nauc_ndcg_at_20_std": 0.416609, "nauc_ndcg_at_20_diff1": 0.456509, "nauc_ndcg_at_100_max": 0.568074, "nauc_ndcg_at_100_std": 0.427956, "nauc_ndcg_at_100_diff1": 0.454214, "nauc_ndcg_at_1000_max": 0.568074, "nauc_ndcg_at_1000_std": 0.427956, "nauc_ndcg_at_1000_diff1": 0.454214, "nauc_map_at_1_max": 0.380086, "nauc_map_at_1_std": 0.137759, "nauc_map_at_1_diff1": 0.53078, "nauc_map_at_3_max": 0.444147, "nauc_map_at_3_std": 0.230839, "nauc_map_at_3_diff1": 0.44514, "nauc_map_at_5_max": 0.473075, "nauc_map_at_5_std": 0.27471, "nauc_map_at_5_diff1": 0.436603, "nauc_map_at_10_max": 0.49582, "nauc_map_at_10_std": 0.315477, "nauc_map_at_10_diff1": 0.429929, "nauc_map_at_20_max": 0.50422, "nauc_map_at_20_std": 0.333656, "nauc_map_at_20_diff1": 0.423574, "nauc_map_at_100_max": 0.506384, "nauc_map_at_100_std": 0.340414, "nauc_map_at_100_diff1": 0.421959, "nauc_map_at_1000_max": 0.506384, "nauc_map_at_1000_std": 0.340414, "nauc_map_at_1000_diff1": 0.421959, "nauc_recall_at_1_max": 0.380086, "nauc_recall_at_1_std": 0.137759, "nauc_recall_at_1_diff1": 0.53078, "nauc_recall_at_3_max": 0.451038, "nauc_recall_at_3_std": 0.259372, "nauc_recall_at_3_diff1": 0.408157, "nauc_recall_at_5_max": 0.495678, "nauc_recall_at_5_std": 0.34057, "nauc_recall_at_5_diff1": 0.397404, "nauc_recall_at_10_max": 0.536355, "nauc_recall_at_10_std": 0.419948, "nauc_recall_at_10_diff1": 0.388233, "nauc_recall_at_20_max": 0.568606, "nauc_recall_at_20_std": 0.473268, "nauc_recall_at_20_diff1": 0.382656, "nauc_recall_at_100_max": 0.579585, "nauc_recall_at_100_std": 0.503516, "nauc_recall_at_100_diff1": 0.379218, "nauc_recall_at_1000_max": 0.579585, "nauc_recall_at_1000_std": 0.503516, "nauc_recall_at_1000_diff1": 0.379218, "nauc_precision_at_1_max": 0.462181, "nauc_precision_at_1_std": 0.289997, "nauc_precision_at_1_diff1": 0.452311, "nauc_precision_at_3_max": 0.371645, "nauc_precision_at_3_std": 0.353764, "nauc_precision_at_3_diff1": 0.15822, "nauc_precision_at_5_max": 0.378867, "nauc_precision_at_5_std": 0.430399, "nauc_precision_at_5_diff1": 0.067078, "nauc_precision_at_10_max": 0.336414, "nauc_precision_at_10_std": 0.437526, "nauc_precision_at_10_diff1": 0.008673, "nauc_precision_at_20_max": 0.277217, "nauc_precision_at_20_std": 0.403684, "nauc_precision_at_20_diff1": -0.03217, "nauc_precision_at_100_max": 0.251104, "nauc_precision_at_100_std": 0.391395, "nauc_precision_at_100_diff1": -0.041993, "nauc_precision_at_1000_max": 0.251104, "nauc_precision_at_1000_std": 0.391395, "nauc_precision_at_1000_diff1": -0.041993, "nauc_mrr_at_1_max": 0.462181, "nauc_mrr_at_1_std": 0.289997, "nauc_mrr_at_1_diff1": 0.452311, "nauc_mrr_at_3_max": 0.531371, "nauc_mrr_at_3_std": 0.37537, "nauc_mrr_at_3_diff1": 0.47127, "nauc_mrr_at_5_max": 0.551749, "nauc_mrr_at_5_std": 0.404133, "nauc_mrr_at_5_diff1": 0.479949, "nauc_mrr_at_10_max": 0.552526, "nauc_mrr_at_10_std": 0.405609, "nauc_mrr_at_10_diff1": 0.482672, "nauc_mrr_at_20_max": 0.552256, "nauc_mrr_at_20_std": 0.406065, "nauc_mrr_at_20_diff1": 0.482101, "nauc_mrr_at_100_max": 0.552108, "nauc_mrr_at_100_std": 0.405884, "nauc_mrr_at_100_diff1": 0.481984, "nauc_mrr_at_1000_max": 0.552108, "nauc_mrr_at_1000_std": 0.405884, "nauc_mrr_at_1000_diff1": 0.481984, "hit_rate_at_1": 0.47802, "hit_rate_at_3": 0.64011, "hit_rate_at_5": 0.7033, "hit_rate_at_10": 0.73901, "hit_rate_at_20": 0.76099, "hit_rate_at_100": 0.76374, "hit_rate_at_1000": 0.76374, "main_score": 0.48045, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.24.7" }, { "ndcg_at_1": 0.45192, "ndcg_at_3": 0.45605, "ndcg_at_5": 0.45681, "ndcg_at_10": 0.46693, "ndcg_at_20": 0.47967, "ndcg_at_100": 0.48726, "ndcg_at_1000": 0.48726, "map_at_1": 0.23696, "map_at_3": 0.33026, "map_at_5": 0.3549, "map_at_10": 0.38094, "map_at_20": 0.39537, "map_at_100": 0.39937, "map_at_1000": 0.39937, "recall_at_1": 0.23696, "recall_at_3": 0.37238, "recall_at_5": 0.4249, "recall_at_10": 0.49213, "recall_at_20": 0.53673, "recall_at_100": 0.55782, "recall_at_1000": 0.55782, "accuracy": 0.23696, "precision_at_1": 0.46978, "precision_at_3": 0.30586, "precision_at_5": 0.23187, "precision_at_10": 0.15412, "precision_at_20": 0.09423, "precision_at_100": 0.02055, "precision_at_1000": 0.00205, "mrr_at_1": 0.46978, "mrr_at_3": 0.54304, "mrr_at_5": 0.555403, "mrr_at_10": 0.561737, "mrr_at_20": 0.56223, "mrr_at_100": 0.562501, "mrr_at_1000": 0.562501, "nauc_ndcg_at_1_max": 0.479327, "nauc_ndcg_at_1_std": 0.262119, "nauc_ndcg_at_1_diff1": 0.510634, "nauc_ndcg_at_3_max": 0.529523, "nauc_ndcg_at_3_std": 0.348208, "nauc_ndcg_at_3_diff1": 0.43216, "nauc_ndcg_at_5_max": 0.554979, "nauc_ndcg_at_5_std": 0.375163, "nauc_ndcg_at_5_diff1": 0.431708, "nauc_ndcg_at_10_max": 0.572096, "nauc_ndcg_at_10_std": 0.399231, "nauc_ndcg_at_10_diff1": 0.425086, "nauc_ndcg_at_20_max": 0.572588, "nauc_ndcg_at_20_std": 0.410308, "nauc_ndcg_at_20_diff1": 0.419573, "nauc_ndcg_at_100_max": 0.576483, "nauc_ndcg_at_100_std": 0.42247, "nauc_ndcg_at_100_diff1": 0.418834, "nauc_ndcg_at_1000_max": 0.576483, "nauc_ndcg_at_1000_std": 0.42247, "nauc_ndcg_at_1000_diff1": 0.418834, "nauc_map_at_1_max": 0.38085, "nauc_map_at_1_std": 0.122994, "nauc_map_at_1_diff1": 0.503176, "nauc_map_at_3_max": 0.446255, "nauc_map_at_3_std": 0.218654, "nauc_map_at_3_diff1": 0.428367, "nauc_map_at_5_max": 0.47658, "nauc_map_at_5_std": 0.260516, "nauc_map_at_5_diff1": 0.42174, "nauc_map_at_10_max": 0.503633, "nauc_map_at_10_std": 0.303548, "nauc_map_at_10_diff1": 0.405015, "nauc_map_at_20_max": 0.510615, "nauc_map_at_20_std": 0.324799, "nauc_map_at_20_diff1": 0.39487, "nauc_map_at_100_max": 0.51229, "nauc_map_at_100_std": 0.331611, "nauc_map_at_100_diff1": 0.393561, "nauc_map_at_1000_max": 0.51229, "nauc_map_at_1000_std": 0.331611, "nauc_map_at_1000_diff1": 0.393561, "nauc_recall_at_1_max": 0.38085, "nauc_recall_at_1_std": 0.122994, "nauc_recall_at_1_diff1": 0.503176, "nauc_recall_at_3_max": 0.446338, "nauc_recall_at_3_std": 0.242651, "nauc_recall_at_3_diff1": 0.406137, "nauc_recall_at_5_max": 0.505042, "nauc_recall_at_5_std": 0.330894, "nauc_recall_at_5_diff1": 0.386116, "nauc_recall_at_10_max": 0.543221, "nauc_recall_at_10_std": 0.407702, "nauc_recall_at_10_diff1": 0.345631, "nauc_recall_at_20_max": 0.565025, "nauc_recall_at_20_std": 0.464396, "nauc_recall_at_20_diff1": 0.319335, "nauc_recall_at_100_max": 0.573145, "nauc_recall_at_100_std": 0.496135, "nauc_recall_at_100_diff1": 0.318656, "nauc_recall_at_1000_max": 0.573145, "nauc_recall_at_1000_std": 0.496135, "nauc_recall_at_1000_diff1": 0.318656, "nauc_precision_at_1_max": 0.483506, "nauc_precision_at_1_std": 0.275491, "nauc_precision_at_1_diff1": 0.499568, "nauc_precision_at_3_max": 0.434047, "nauc_precision_at_3_std": 0.42222, "nauc_precision_at_3_diff1": 0.149084, "nauc_precision_at_5_max": 0.410998, "nauc_precision_at_5_std": 0.453293, "nauc_precision_at_5_diff1": 0.080463, "nauc_precision_at_10_max": 0.3539, "nauc_precision_at_10_std": 0.463234, "nauc_precision_at_10_diff1": -0.003413, "nauc_precision_at_20_max": 0.295891, "nauc_precision_at_20_std": 0.448188, "nauc_precision_at_20_diff1": -0.045624, "nauc_precision_at_100_max": 0.272708, "nauc_precision_at_100_std": 0.447661, "nauc_precision_at_100_diff1": -0.059918, "nauc_precision_at_1000_max": 0.272708, "nauc_precision_at_1000_std": 0.447661, "nauc_precision_at_1000_diff1": -0.059918, "nauc_mrr_at_1_max": 0.483506, "nauc_mrr_at_1_std": 0.275491, "nauc_mrr_at_1_diff1": 0.499568, "nauc_mrr_at_3_max": 0.536508, "nauc_mrr_at_3_std": 0.353256, "nauc_mrr_at_3_diff1": 0.501247, "nauc_mrr_at_5_max": 0.562454, "nauc_mrr_at_5_std": 0.386036, "nauc_mrr_at_5_diff1": 0.501109, "nauc_mrr_at_10_max": 0.56405, "nauc_mrr_at_10_std": 0.393165, "nauc_mrr_at_10_diff1": 0.500662, "nauc_mrr_at_20_max": 0.564845, "nauc_mrr_at_20_std": 0.394864, "nauc_mrr_at_20_diff1": 0.500347, "nauc_mrr_at_100_max": 0.565031, "nauc_mrr_at_100_std": 0.395536, "nauc_mrr_at_100_diff1": 0.500305, "nauc_mrr_at_1000_max": 0.565031, "nauc_mrr_at_1000_std": 0.395536, "nauc_mrr_at_1000_diff1": 0.500305, "hit_rate_at_1": 0.46978, "hit_rate_at_3": 0.62637, "hit_rate_at_5": 0.67857, "hit_rate_at_10": 0.72253, "hit_rate_at_20": 0.73077, "hit_rate_at_100": 0.73901, "hit_rate_at_1000": 0.73901, "main_score": 0.46693, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.24.7" } ] }, "evaluation_time": 1087.694951541991, "kg_co2_emissions": null, "date": 1791750870.366308, "evaluation_phases": [ { "name": "Data loading", "start": 4.526998964138329e-06, "end": 74.34512634600105, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 74.34514129899617, "end": 74.34514387299714, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 74.35071620999952, "end": 245.6836313429958, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 245.68366817799688, "end": 245.72890802899929, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 245.74090456699923, "end": 420.2684895400016, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 420.2685226600006, "end": 420.31506923499546, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 420.32772204499634, "end": 589.9446896609952, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 589.9447215190012, "end": 589.9908600339986, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 590.004435062001, "end": 759.5539032659944, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 759.5539486839989, "end": 759.6027002039991, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 759.6165007699965, "end": 924.3963623519958, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 924.3963981259949, "end": 924.4419883719966, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 924.4557917009952, "end": 1087.638966710998, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 1087.6390103660015, "end": 1087.684890976001, "split": "test", "subset": "portuguese" } ] }