Merge branch 'main' into tasklist

# Conflicts:
#	pyproject.toml

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

@@ -26,5 +26,3 @@ metric_list:
     ignore_punctuation: true
 metadata:
   version: 2.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml

@@ -34,5 +34,3 @@ metric_list:
     ignore_punctuation: true
 metadata:
   version: 3.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu/flan_cot_zeroshot/utils.py

@@ -17,7 +17,7 @@ class MultiChoiceRegexFilter(RegexFilter):
         ignore_punctuation=False,
         regexes_to_ignore=None,
     ) -> None:
-        """
+        r"""
         regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure
                         - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response.
                         - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices.
@@ -90,7 +90,7 @@ class MultiChoiceRegexFilter(RegexFilter):
             fallback_regex = re.compile("|".join(fallback_regexes))
             without_paren_fallback_regex = "|".join(without_paren_fallback_regexes)
             without_paren_fallback_regex = re.compile(
-                f":[\s]*({without_paren_fallback_regex})"
+                rf":[\s]*({without_paren_fallback_regex})"
             )
 
             filtered = []

lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml

@@ -30,5 +30,3 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 3.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py

@@ -17,7 +17,7 @@ class MultiChoiceRegexFilter(RegexFilter):
         ignore_punctuation=False,
         regexes_to_ignore=None,
     ) -> None:
-        """
+        r"""
         regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure
                         - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response.
                         - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices.
@@ -90,7 +90,7 @@ class MultiChoiceRegexFilter(RegexFilter):
             fallback_regex = re.compile("|".join(fallback_regexes))
             without_paren_fallback_regex = "|".join(without_paren_fallback_regexes)
             without_paren_fallback_regex = re.compile(
-                f":[\s]*({without_paren_fallback_regex})"
+                rf":[\s]*({without_paren_fallback_regex})"
             )
 
             filtered = []

lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml

@@ -13,5 +13,3 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 2.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu/generative/_default_template_yaml

@@ -30,5 +30,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 3.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu_prox/mmlu_prox_config_generator.py

@@ -44,7 +44,7 @@ if __name__ == "__main__":
                     line = line.format(lang=lang_abbr)
                 if "{ans_regex}" in line:
                     ans_regex = lang_lib_list[-1].replace(
-                        "({})", "\(?([ABCDEFGHIJ])\)?"
+                        "({})", r"\(?([ABCDEFGHIJ])\)?"
                     )
                     if lang_abbr == "en":
                         ans_regex = ans_regex.lstrip("the").strip()

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

@@ -12,5 +12,3 @@ metric_list:
   - metric: acc
 metadata:
   version: 0.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

@@ -12,5 +12,3 @@ metric_list:
   - metric: acc
 metadata:
   version: 0.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

@@ -12,5 +12,3 @@ metric_list:
   - metric: acc
 metadata:
   version: 0.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mts_dialog/utils.py

@@ -11,7 +11,9 @@ try:
 
 except (ModuleNotFoundError, ImportError):
     raise ModuleNotFoundError(
-        "Please install evaluation metrics via pip install evaluate and pip install bert-score",
+        "Please install evaluation metrics via pip install evaluate bert-score "
+        "rouge_score>=0.1.2 nltk absl-py "
+        "git+https://github.com/google-research/bleurt.git"
     )
 except Exception as e:
     raise RuntimeError(

lm_eval/tasks/multiblimp/README.md

+# MultiBLiMP: A Massively Multilingual Benchmark of Linguistic Minimal Pairs
+
+## Task Description
+MultiBLiMP is a massively multilingual benchmark of linguistic minimal pairs, covering 101 languages, 6 linguistic phenomena and containing more than 125,000 minimal pairs.
+
+* Paper: https://arxiv.org/abs/2504.02768
+* GitHub Repo: https://github.com/jumelet/multiblimp/
+* Hugging Face Dataset Repo: https://huggingface.co/datasets/jumelet/multiblimp
+
+## Implementation
+
+* `multiblimp_{lang}` runs MultiBLiMP for a given language, where `{lang}` must be replaced by the language's ISO 639-3 code (e.g., `eng` for English, `abk` for Abkhazian, `wbp` for Warlpiri, etc.).
+* `multiblimp` tag runs MultiBLiMP for all languages
+
+Note: The original implementation is provided [here](https://github.com/jumelet/multiblimp), and the [dataset repository](https://huggingface.co/datasets/jumelet/multiblimp) also includes a link to a more flexible version of the implementation [here](https://github.com/catherinearnett/multiblimp). This implementation follows these as closely as possible, but the original implementations normalize length by number of tokens, which is not supported by the Language Model Evaluation Harness (see [[1](https://blog.eleuther.ai/multiple-choice-normalization/)], [[2](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md)], [[3](https://github.com/EleutherAI/lm-evaluation-harness/issues/1396)]). For this reason, the implementation provided here includes both the `acc` (accuracy based on comparing the unnormalized log-probability of the correct and incorrect versions of each sentence) and `acc_norm` (the same as `acc` but with sentence log-probability normalized by number of bytes) metrics.
+
+## Dataset Details
+
+This table (from the [Hugging Face Dataset Repo](https://huggingface.co/datasets/jumelet/multiblimp)) lists the languages covered in MultiBLiMP and the number of items for each language.
+
+| ISO Code |      Language      |   n  |
+|:--------:|:------------------:|:----:|
+| abk      | Abkhazian          | 40   |
+| aqz      | Akuntsu            | 14   |
+| sqi      | Albanian           | 243  |
+| amh      | Amharic            | 112  |
+| grc      | Ancient Greek      | 3695 |
+| hbo      | Ancient Hebrew     | 983  |
+| apu      | Apurinã            | 28   |
+| hye      | Armenian           | 1415 |
+| eus      | Basque             | 273  |
+| bel      | Belarusian         | 2570 |
+| ben      | Bengali            | 21   |
+| bho      | Bhojpuri           | 34   |
+| bor      | Borôro             | 241  |
+| bre      | Breton             | 260  |
+| bul      | Bulgarian          | 2458 |
+| bua      | Buriat             | 103  |
+| cat      | Catalan            | 2284 |
+| chu      | Church Slavonic    | 4166 |
+| xcl      | Classical Armenian | 1623 |
+| ces      | Czech              | 4256 |
+| dan      | Danish             | 50   |
+| nld      | Dutch              | 2331 |
+| egy      | Egyptian (Ancient) | 22   |
+| eng      | English            | 770  |
+| myv      | Erzya              | 464  |
+| est      | Estonian           | 2575 |
+| fao      | Faroese            | 232  |
+| fin      | Finnish            | 2570 |
+| fra      | French             | 2548 |
+| glg      | Galician           | 753  |
+| kat      | Georgian           | 204  |
+| deu      | German             | 2298 |
+| aln      | Gheg Albanian      | 677  |
+| got      | Gothic             | 1579 |
+| guj      | Gujarati           | 7    |
+| heb      | Hebrew             | 2330 |
+| azz      | H-P Nahuatl        | 207  |
+| hin      | Hindi              | 1447 |
+| hit      | Hittite            | 50   |
+| hun      | Hungarian          | 845  |
+| isl      | Icelandic          | 2801 |
+| gle      | Irish              | 28   |
+| ita      | Italian            | 2999 |
+| quc      | K'iche'            | 131  |
+| xnr      | Kangri             | 86   |
+| krl      | Karelian           | 260  |
+| kxh      | Karo (Ethiopia)    | 120  |
+| kaz      | Kazakh             | 173  |
+| kir      | Kirghiz            | 185  |
+| koi      | Komi-Permyak       | 43   |
+| kpv      | Komi-Zyrian        | 320  |
+| lat      | Latin              | 3149 |
+| lav      | Latvian            | 3032 |
+| lij      | Ligurian           | 254  |
+| lit      | Lithuanian         | 1180 |
+| olo      | Livvi              | 190  |
+| nds      | Low German         | 1774 |
+| mkd      | Macedonian         | 39   |
+| mar      | Marathi            | 460  |
+| frm      | Middle French      | 294  |
+| ell      | Modern Greek       | 1096 |
+| mdf      | Moksha             | 82   |
+| yrl      | Nhengatu           | 720  |
+| pcm      | Nigerian Pidgin    | 26   |
+| kmr      | Northern Kurdish   | 544  |
+| sme      | Northern Sami      | 2536 |
+| fro      | Old French         | 1976 |
+| orv      | Old Russian        | 4615 |
+| ota      | Ottoman Turkish    | 99   |
+| fas      | Persian            | 2553 |
+| xpg      | Phrygian           | 50   |
+| pol      | Polish             | 3272 |
+| por      | Portuguese         | 3048 |
+| ron      | Romanian           | 2056 |
+| rus      | Russian            | 3832 |
+| san      | Sanskrit           | 4442 |
+| gla      | Scottish Gaelic    | 66   |
+| hbs      | Serbo-Croatian     | 3286 |
+| sms      | Skolt Sami         | 263  |
+| slk      | Slovak             | 4145 |
+| slv      | Slovenian          | 4483 |
+| spa      | Spanish            | 2541 |
+| arb      | Standard Arabic    | 1215 |
+| swe      | Swedish            | 201  |
+| tam      | Tamil              | 382  |
+| ttc      | Tektiteko          | 69   |
+| tpn      | Tupinambá          | 9    |
+| tur      | Turkish            | 1742 |
+| uig      | Uighur             | 758  |
+| ukr      | Ukrainian          | 2744 |
+| hsb      | Upper Sorbian      | 186  |
+| urd      | Urdu               | 550  |
+| urb      | Urubú-Kaapor       | 13   |
+| uzb      | Uzbek              | 50   |
+| vep      | Veps               | 187  |
+| wbp      | Warlpiri           | 12   |
+| cym      | Welsh              | 1120 |
+| hyw      | Western Armenian   | 1153 |
+| wol      | Wolof              | 705  |
+| sah      | Yakut              | 144  |
+| nhi      | Tenango Nahuatl    | 38   |
+
+
+## Citation
+```
+@misc{jumelet2025multiblimp10massivelymultilingual,
+      title={MultiBLiMP 1.0: A Massively Multilingual Benchmark of Linguistic Minimal Pairs},
+      author={Jaap Jumelet and Leonie Weissweiler and Arianna Bisazza},
+      year={2025},
+      eprint={2504.02768},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2504.02768},
+}
+```
+
+## New Task Checklist
+
+- [x] Is the task an existing benchmark in the literature?
+  - [x] Have you referenced the original paper that introduced the task?
+  - [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?

lm_eval/tasks/multiblimp/_template_yaml

+dataset_path: jumelet/multiblimp
+tag: multiblimp
+output_type: multiple_choice
+test_split: train
+doc_to_text: ""
+target_delimiter: ""
+doc_to_target: 0
+doc_to_choice: "{{[sen,wrong_sen]}}"
+num_fewshot: 0
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0

lm_eval/tasks/multiblimp/multiblimp_abk.yaml

+include: _template_yaml
+dataset_name: abk
+task: multiblimp_abk

lm_eval/tasks/multiblimp/multiblimp_aln.yaml

+include: _template_yaml
+dataset_name: aln
+task: multiblimp_aln

lm_eval/tasks/multiblimp/multiblimp_amh.yaml

+include: _template_yaml
+dataset_name: amh
+task: multiblimp_amh

lm_eval/tasks/multiblimp/multiblimp_apu.yaml

+include: _template_yaml
+dataset_name: apu
+task: multiblimp_apu

lm_eval/tasks/multiblimp/multiblimp_aqz.yaml

+include: _template_yaml
+dataset_name: aqz
+task: multiblimp_aqz

lm_eval/tasks/multiblimp/multiblimp_arb.yaml

+include: _template_yaml
+dataset_name: arb
+task: multiblimp_arb