quick and dirty implementation of m-mmlu

lm_eval/tasks/m_mmlu/_default_template_yaml

+dataset_path: openai/MMMLU
+dataset_name: by_language
+tag: mmmlu
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{Question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: Answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/m_mmlu/arabic.yaml

+include: _default_template_yaml
+task: mmmlu_arabic
+test_split: AR_XY
+fewshot_split: AR_XY

lm_eval/tasks/m_mmlu/bengali.yaml

+include: _default_template_yaml
+task: mmmlu_bengali
+test_split: BN_BD
+fewshot_split: BN_BD

lm_eval/tasks/m_mmlu/brazilian_portuguese.yaml

+include: _default_template_yaml
+task: mmmlu_brazilian_portuguese
+test_split: PT_BR
+fewshot_split: PT_BR

lm_eval/tasks/m_mmlu/french.yaml

+include: _default_template_yaml
+task: mmmlu_french
+test_split: FR_FR
+fewshot_split: FR_FR

lm_eval/tasks/m_mmlu/german.yaml

+include: _default_template_yaml
+task: mmmlu_german
+test_split: DE_DE
+fewshot_split: DE_DE

lm_eval/tasks/m_mmlu/hindi.yaml

+include: _default_template_yaml
+task: mmmlu_hindi
+test_split: HI_IN
+fewshot_split: HI_IN

lm_eval/tasks/m_mmlu/indonesian.yaml

+include: _default_template_yaml
+task: mmmlu_indonesian
+test_split: ID_ID
+fewshot_split: ID_ID

lm_eval/tasks/m_mmlu/italian.yaml

+include: _default_template_yaml
+task: mmmlu_italian
+test_split: IT_IT
+fewshot_split: IT_IT

lm_eval/tasks/m_mmlu/japanese.yaml

+include: _default_template_yaml
+task: mmmlu_japanese
+test_split: JA_JP
+fewshot_split: JA_JP

lm_eval/tasks/m_mmlu/korean.yaml

+include: _default_template_yaml
+task: mmmlu_korean
+test_split: KO_KR
+fewshot_split: KO_KR

lm_eval/tasks/m_mmlu/simplified_chinese.yaml

+include: _default_template_yaml
+task: mmmlu_chinese
+test_split: ZH_CN
+fewshot_split: ZH_CN

lm_eval/tasks/m_mmlu/spanish.yaml

+include: _default_template_yaml
+task: mmmlu_spanish
+test_split: ES_LA
+fewshot_split: ES_LA

lm_eval/tasks/m_mmlu/swahili.yaml

+include: _default_template_yaml
+task: mmmlu_swahili
+test_split: SW_KE
+fewshot_split: SW_KE

lm_eval/tasks/m_mmlu/yoruba.yaml

+include: _default_template_yaml
+task: mmmlu_yoruba
+test_split: YO_NG
+fewshot_split: YO_NG