Merge branch 'main' into metrics

# Conflicts:
#	lm_eval/models/vllm_causallms.py
#	pyproject.toml

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_professional_medicine.yaml

+"dataset_name": "professional_medicine"
+"description":
+  "The following are multiple choice questions (with answers) about professional\
+  \ medicine.\n\n"
+"tag": "mmlu_other_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_professional_medicine_generative_spanish"
+"task_alias": "professional_medicine_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_professional_psychology.yaml

+"dataset_name": "professional_psychology"
+"description":
+  "The following are multiple choice questions (with answers) about professional\
+  \ psychology.\n\n"
+"tag": "mmlu_social_sciences_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_professional_psychology_generative_spanish"
+"task_alias": "professional_psychology_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_public_relations.yaml

+"dataset_name": "public_relations"
+"description":
+  "The following are multiple choice questions (with answers) about public\
+  \ relations.\n\n"
+"tag": "mmlu_social_sciences_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_public_relations_generative_spanish"
+"task_alias": "public_relations_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_security_studies.yaml

+"dataset_name": "security_studies"
+"description":
+  "The following are multiple choice questions (with answers) about security\
+  \ studies.\n\n"
+"tag": "mmlu_social_sciences_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_security_studies_generative_spanish"
+"task_alias": "security_studies_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_sociology.yaml

+"dataset_name": "sociology"
+"description":
+  "The following are multiple choice questions (with answers) about sociology.\n\
+  \n"
+"tag": "mmlu_social_sciences_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_sociology_generative_spanish"
+"task_alias": "sociology_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_us_foreign_policy.yaml

+"dataset_name": "us_foreign_policy"
+"description":
+  "The following are multiple choice questions (with answers) about us\
+  \ foreign policy.\n\n"
+"tag": "mmlu_social_sciences_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_us_foreign_policy_generative_spanish"
+"task_alias": "us_foreign_policy_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_virology.yaml

+"dataset_name": "virology"
+"description":
+  "The following are multiple choice questions (with answers) about virology.\n\
+  \n"
+"tag": "mmlu_other_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_virology_generative_spanish"
+"task_alias": "virology_spanish"

lm_eval/tasks/mmlu-redux-spanish/generative/mmlu_world_religions.yaml

+"dataset_name": "world_religions"
+"description":
+  "The following are multiple choice questions (with answers) about world\
+  \ religions.\n\n"
+"tag": "mmlu_humanities_generative_spanish"
+"include": "_default_template_spanish_yaml"
+"task": "mmlu_world_religions_generative_spanish"
+"task_alias": "world_religions_spanish"

lm_eval/tasks/mmlu-redux-spanish/mmlu-redux-2.0-spanish.yaml

+task: "mmlu_redux_spanish"
+dataset_path: amias-mx/mmlu-redux-2.0-spanish
+dataset_name: abstract_algebra
+test_split: test
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/mmlu-redux/generative/README.md

+# Task-name
+
+### Paper
+
+Title: `Are We Donewith MMLU?`
+
+Abstract: `https://arxiv.org/pdf/2406.04127`
+
+`The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.`
+
+Homepage: `https://huggingface.co/datasets/edinburgh-dawg/mmlu-redux-2.0`
+
+### Citation
+
+```
+BibTeX
+@misc{edinburgh2024mmlu,
+      title={Are We Done with MMLU?},
+      author={Aryo Pradipta Gema and Joshua Ong Jun Leang and Giwon Hong and Alessio Devoto and
+      Alberto Carlo Maria Mancino and Rohit Saxena and Xuanli He and Yu Zhao and Xiaotang Du and
+      MohammadRezaGhasemi Madani and Claire Barale and Robert McHardy and Joshua Harris and
+      Jean Kaddour and Emile van Krieken and Pasquale Minervini},
+      year={2025},
+      eprint={2406.04127},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+### Groups, Tags, and Tasks
+
+#### Groups
+
+- `stem`
+- `other`
+- `social sciences`
+- `humanities`
+
+#### Tasks
+
+- `mmlu_stem_generative`
+- `mmlu_other_generative`
+- `mmlu_social_sciences_generative`
+- `mmlu_humanities_generative`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+
+- [x] Is the task an existing benchmark in the literature?
+  - [x] Have you referenced the original paper that introduced the task?
+  - [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+If other tasks on this dataset are already supported:
+
+- [ ] Is the "Main" variant of this task clearly denoted?
+- [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+- [ ] Have you noted which, if any, published evaluation setups are matched by this variant?
+
+ver 1: PR #2705
+First implementation

lm_eval/tasks/mmlu-redux/generative/_default_template_yaml

+dataset_path: "edinburgh-dawg/mmlu-redux-2.0"
+test_split: test
+dataset_kwargs:
+  trust_remote_code: true
+
+output_type: generate_until
+
+doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nPlease respond with the correct letter (A, B, C or D) without any additional comments, only the correct letter:"
+doc_to_target: "{{['A','B','C','D'][answer]}}"
+target_delimiter: ":"
+generation_kwargs:
+  until:
+    - "</s>"
+
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+
+# IMPORTANT: rename your filter to "default" so older harness automatically applies it.
+filter_list:
+  - name: default
+    filter:
+      # This captures the first single capital letter A/B/C/D
+      - function: regex
+        regex_pattern: "([ABCD])"
+      - function: take_first
+
+metadata:
+  version: 3.0

lm_eval/tasks/mmlu-redux/generative/_mmlu.yaml

+group: mmlu_redux_generative
+group_alias: mmlu_redux (generative)
+task:
+  - group: stem
+    task:
+      - mmlu_stem_generative
+    aggregate_metric_list:
+      - metric: exact_match
+        weight_by_size: true
+  - group: other
+    task:
+      - mmlu_other_generative
+    aggregate_metric_list:
+      - metric: exact_match
+        weight_by_size: true
+  - group: social sciences
+    task:
+      - mmlu_social_sciences_generative
+    aggregate_metric_list:
+      - metric: exact_match
+        weight_by_size: true
+  - group: humanities
+    task:
+      - mmlu_humanities_generative
+    aggregate_metric_list:
+      - metric: exact_match
+        weight_by_size: true
+aggregate_metric_list:
+  - aggregation: mean
+    metric: exact_match
+    weight_by_size: true
+metadata:
+  version: 3

lm_eval/tasks/mmlu-redux/generative/mmlu_abstract_algebra.yaml

+"dataset_name": "abstract_algebra"
+"description": "The following are multiple choice questions (with answers) about abstract\
+  \ algebra.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_abstract_algebra_generative"
+"task_alias": "abstract_algebra"

lm_eval/tasks/mmlu-redux/generative/mmlu_anatomy.yaml

+"dataset_name": "anatomy"
+"description": "The following are multiple choice questions (with answers) about anatomy.\n\
+  \n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_anatomy_generative"
+"task_alias": "anatomy"

lm_eval/tasks/mmlu-redux/generative/mmlu_astronomy.yaml

+"dataset_name": "astronomy"
+"description": "The following are multiple choice questions (with answers) about astronomy.\n\
+  \n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_astronomy_generative"
+"task_alias": "astronomy"

lm_eval/tasks/mmlu-redux/generative/mmlu_business_ethics.yaml

+"dataset_name": "business_ethics"
+"description": "The following are multiple choice questions (with answers) about business\
+  \ ethics.\n\n"
+"tag": "mmlu_other_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_business_ethics_generative"
+"task_alias": "business_ethics"

lm_eval/tasks/mmlu-redux/generative/mmlu_clinical_knowledge.yaml

+"dataset_name": "clinical_knowledge"
+"description": "The following are multiple choice questions (with answers) about clinical\
+  \ knowledge.\n\n"
+"tag": "mmlu_other_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_clinical_knowledge_generative"
+"task_alias": "clinical_knowledge"

lm_eval/tasks/mmlu-redux/generative/mmlu_college_biology.yaml

+"dataset_name": "college_biology"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ biology.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_college_biology_generative"
+"task_alias": "college_biology"

lm_eval/tasks/mmlu-redux/generative/mmlu_college_chemistry.yaml

+"dataset_name": "college_chemistry"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ chemistry.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_college_chemistry_generative"
+"task_alias": "college_chemistry"

lm_eval/tasks/mmlu-redux/generative/mmlu_college_computer_science.yaml

+"dataset_name": "college_computer_science"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ computer science.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_college_computer_science_generative"
+"task_alias": "college_computer_science"