Merge branch 'main' into mathvista

# Conflicts:
#	lm_eval/models/openai_completions.py

lm_eval/tasks/mlqa/generate_tasks.py

+# ruff: noqa: E731, E741
+"""
+Script to generate task YAMLs for the mlqa dataset.
+Based on `tasks/bigbench/generate_tasks.py`.
+"""
+
+from datasets import get_dataset_config_names
+
+
+chosen_subtasks = []
+
+language_dict = {
+    "en": "english",
+    "es": "spanish",
+    "hi": "hindi",
+    "vi": "vietnamese",
+    "de": "german",
+    "ar": "arabic",
+    "zh": "chinese",
+}
+
+
+def main() -> None:
+    configs = get_dataset_config_names("facebook/mlqa", trust_remote_code=True)
+    for config in configs:
+        if len(config.split(".")) == 2:
+            continue
+        else:
+            chosen_subtasks.append(config)
+    assert len(chosen_subtasks) == 49
+    for task in chosen_subtasks:
+        file_name = f"{task.replace('.', '_')}.yaml"
+        context_lang = file_name.split("_")[1]
+        # Not using yaml to avoid tagging issues with !function
+        with open(file_name, "w", encoding="utf-8") as f:
+            f.write("# Generated by generate_tasks.py\n")
+
+            # Manually writing the YAML-like content inside files to avoid tagging issues
+            f.write("include: mlqa_common_yaml\n")
+            f.write(f"task: {task.replace('.', '_')}\n")
+            f.write(f"dataset_name: {task}\n")
+            f.write(
+                f"process_results: !function utils.process_results_{context_lang}\n"
+            )
+
+
+if __name__ == "__main__":
+    main()

lm_eval/tasks/mlqa/mlqa_ar_ar.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_ar
+dataset_name: mlqa.ar.ar
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_ar_de.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_de
+dataset_name: mlqa.ar.de
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_ar_en.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_en
+dataset_name: mlqa.ar.en
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_ar_es.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_es
+dataset_name: mlqa.ar.es
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_ar_hi.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_hi
+dataset_name: mlqa.ar.hi
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_ar_vi.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_vi
+dataset_name: mlqa.ar.vi
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_ar_zh.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_ar_zh
+dataset_name: mlqa.ar.zh
+process_results: !function utils.process_results_ar

lm_eval/tasks/mlqa/mlqa_common_yaml

+dataset_path: facebook/mlqa
+dataset_kwargs:
+  trust_remote_code: true
+test_split: test
+validation_split: validation
+output_type: generate_until
+doc_to_text: "Context: {{context}}\n\nQuestion: {{question}}\n\nAnswer:"
+doc_to_target: "{{answers}}"
+process_docs: !function utils.process_docs
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+  - metric: f1
+    aggregation: mean
+    higher_is_better: true
+generation_kwargs:
+  until:
+    - "\n"
+  do_sample: false
+metadata:
+  version: 0.0

lm_eval/tasks/mlqa/mlqa_de_ar.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_ar
+dataset_name: mlqa.de.ar
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_de_de.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_de
+dataset_name: mlqa.de.de
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_de_en.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_en
+dataset_name: mlqa.de.en
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_de_es.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_es
+dataset_name: mlqa.de.es
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_de_hi.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_hi
+dataset_name: mlqa.de.hi
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_de_vi.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_vi
+dataset_name: mlqa.de.vi
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_de_zh.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_de_zh
+dataset_name: mlqa.de.zh
+process_results: !function utils.process_results_de

lm_eval/tasks/mlqa/mlqa_en_ar.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_en_ar
+dataset_name: mlqa.en.ar
+process_results: !function utils.process_results_en

lm_eval/tasks/mlqa/mlqa_en_de.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_en_de
+dataset_name: mlqa.en.de
+process_results: !function utils.process_results_en

lm_eval/tasks/mlqa/mlqa_en_en.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_en_en
+dataset_name: mlqa.en.en
+process_results: !function utils.process_results_en

lm_eval/tasks/mlqa/mlqa_en_es.yaml

+# Generated by generate_tasks.py
+include: mlqa_common_yaml
+task: mlqa_en_es
+dataset_name: mlqa.en.es
+process_results: !function utils.process_results_en