add mc tasks

lm_eval/tasks/mrl/mc/_generate_config.py

+from pathlib import Path
+
+import datasets
+
+
+if __name__ == "__main__":
+    subsets = [
+        x
+        for x in datasets.get_dataset_config_names(
+            "mrlbenchmarks/global-piqa-nonparallel"
+        )
+        if not x.startswith("dev")
+    ]
+    PARENT = Path(__file__).parent
+    for s in subsets:
+        with open(PARENT / f"{s}.yaml", "w") as f:
+            f.write("include: '_template_mc'\n")
+            f.write(f"task: mrl_{s}\n")
+            f.write(f"dataset_name: {s}\n")

lm_eval/tasks/mrl/mc/_template_mc

+dataset_path: mrlbenchmarks/global-piqa-nonparallel
+output_type: multiple_choice
+test_split: test
+doc_to_text: prompt
+doc_to_target: label
+doc_to_choice: "{{[solution0, solution1]}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/mrl/mc/acq_arab.yaml

+include: '_template_mc'
+task: mrl_acq_arab
+dataset_name: acq_arab

lm_eval/tasks/mrl/mc/aeb_arab.yaml

+include: '_template_mc'
+task: mrl_aeb_arab
+dataset_name: aeb_arab

lm_eval/tasks/mrl/mc/afb_arab.yaml

+include: '_template_mc'
+task: mrl_afb_arab
+dataset_name: afb_arab

lm_eval/tasks/mrl/mc/als_latn.yaml

+include: '_template_mc'
+task: mrl_als_latn
+dataset_name: als_latn

lm_eval/tasks/mrl/mc/amh_ethi.yaml

+include: '_template_mc'
+task: mrl_amh_ethi
+dataset_name: amh_ethi

lm_eval/tasks/mrl/mc/apc_arab_jord.yaml

+include: '_template_mc'
+task: mrl_apc_arab_jord
+dataset_name: apc_arab_jord

lm_eval/tasks/mrl/mc/apc_arab_leba.yaml

+include: '_template_mc'
+task: mrl_apc_arab_leba
+dataset_name: apc_arab_leba

lm_eval/tasks/mrl/mc/apc_arab_pale.yaml

+include: '_template_mc'
+task: mrl_apc_arab_pale
+dataset_name: apc_arab_pale

lm_eval/tasks/mrl/mc/apc_arab_syri.yaml

+include: '_template_mc'
+task: mrl_apc_arab_syri
+dataset_name: apc_arab_syri

lm_eval/tasks/mrl/mc/arb_arab.yaml

+include: '_template_mc'
+task: mrl_arb_arab
+dataset_name: arb_arab

lm_eval/tasks/mrl/mc/arq_arab.yaml

+include: '_template_mc'
+task: mrl_arq_arab
+dataset_name: arq_arab

lm_eval/tasks/mrl/mc/ars_arab.yaml

+include: '_template_mc'
+task: mrl_ars_arab
+dataset_name: ars_arab

lm_eval/tasks/mrl/mc/ary_arab.yaml

+include: '_template_mc'
+task: mrl_ary_arab
+dataset_name: ary_arab

lm_eval/tasks/mrl/mc/arz_arab.yaml

+include: '_template_mc'
+task: mrl_arz_arab
+dataset_name: arz_arab

lm_eval/tasks/mrl/mc/azj_latn.yaml

+include: '_template_mc'
+task: mrl_azj_latn
+dataset_name: azj_latn

lm_eval/tasks/mrl/mc/bam_latn.yaml

+include: '_template_mc'
+task: mrl_bam_latn
+dataset_name: bam_latn

lm_eval/tasks/mrl/mc/bel_cyrl.yaml

+include: '_template_mc'
+task: mrl_bel_cyrl
+dataset_name: bel_cyrl

lm_eval/tasks/mrl/mc/ben_latn.yaml

+include: '_template_mc'
+task: mrl_ben_latn
+dataset_name: ben_latn