[m_mmul] added multilingual evaluation from alexandrainst/m_mmlu (#1358)

lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml

+group:
+  - m_mmlu
+dataset_path: alexandrainst/m_mmlu
+test_split: test
+fewshot_split: train
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py

+import yaml
+import datasets
+
+from tqdm import tqdm
+
+
+def main() -> None:
+    dataset_path = "alexandrainst/m_mmlu"
+
+    # Removed hy and sk subdataset because the original dataset is broken
+    # I created this PR https://huggingface.co/datasets/alexandrainst/m_mmlu/discussions/3
+    # on the dataset for the authors, in case it will be accepeted the filter can be removed
+    keys_without_hy_sk = list(filter(lambda k: ('hy' not in k and  'sk' not in k), 
+                                     datasets.get_dataset_infos(dataset_path).keys()))
+
+    for task in tqdm():
+        file_name = f"m_mmlu_{task}.yaml"
+        try:
+            with open(f"{file_name}", "w") as f:
+                f.write("# Generated by _generate_configs.py\n")
+                yaml.dump(
+                    {
+                        "include": "_default_yaml",
+                        "task": f"{dataset_path.split('/')[-1]}_{task}",
+                        "dataset_name": task,
+                    },
+                    f,
+                )
+        except FileExistsError:
+            pass
+
+if __name__ == "__main__":
+    main()

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ar.yaml

+# Generated by _generate_configs.py
+dataset_name: ar
+include: _default_yaml
+task: m_mmlu_ar

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml

+# Generated by _generate_configs.py
+dataset_name: bn
+include: _default_yaml
+task: m_mmlu_bn

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ca.yaml

+# Generated by _generate_configs.py
+dataset_name: ca
+include: _default_yaml
+task: m_mmlu_ca

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml

+# Generated by _generate_configs.py
+dataset_name: da
+include: _default_yaml
+task: m_mmlu_da

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_de.yaml

+# Generated by _generate_configs.py
+dataset_name: de
+include: _default_yaml
+task: m_mmlu_de

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_en.yaml

+# Generated by _generate_configs.py
+dataset_name: en
+include: _default_yaml
+task: m_mmlu_en

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml

+# Generated by _generate_configs.py
+dataset_name: es
+include: _default_yaml
+task: m_mmlu_es

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_eu.yaml

+# Generated by _generate_configs.py
+dataset_name: eu
+include: _default_yaml
+task: m_mmlu_eu

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_fr.yaml

+# Generated by _generate_configs.py
+dataset_name: fr
+include: _default_yaml
+task: m_mmlu_fr

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_gu.yaml

+# Generated by _generate_configs.py
+dataset_name: gu
+include: _default_yaml
+task: m_mmlu_gu

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml

+# Generated by _generate_configs.py
+dataset_name: hi
+include: _default_yaml
+task: m_mmlu_hi

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml

+# Generated by _generate_configs.py
+dataset_name: hr
+include: _default_yaml
+task: m_mmlu_hr

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml

+# Generated by _generate_configs.py
+dataset_name: hu
+include: _default_yaml
+task: m_mmlu_hu

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml

+# Generated by _generate_configs.py
+dataset_name: id
+include: _default_yaml
+task: m_mmlu_id

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml

+# Generated by _generate_configs.py
+dataset_name: is
+include: _default_yaml
+task: m_mmlu_is

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml

+# Generated by _generate_configs.py
+dataset_name: it
+include: _default_yaml
+task: m_mmlu_it

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml

+# Generated by _generate_configs.py
+dataset_name: kn
+include: _default_yaml
+task: m_mmlu_kn

lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml

+# Generated by _generate_configs.py
+dataset_name: ml
+include: _default_yaml
+task: m_mmlu_ml