Merge branch 'main' into llama

lm_eval/tasks/global_mmlu/default/global_mmlu_bn.yaml

-# Generated by _generate_configs.py
-dataset_name: bn
-include: _default_yaml
-task: global_mmlu_bn

lm_eval/tasks/global_mmlu/default/global_mmlu_de.yaml

-# Generated by _generate_configs.py
-dataset_name: de
-include: _default_yaml
-task: global_mmlu_de

lm_eval/tasks/global_mmlu/default/global_mmlu_en.yaml

-# Generated by _generate_configs.py
-dataset_name: en
-include: _default_yaml
-task: global_mmlu_en

lm_eval/tasks/global_mmlu/default/global_mmlu_es.yaml

-# Generated by _generate_configs.py
-dataset_name: es
-include: _default_yaml
-task: global_mmlu_es

lm_eval/tasks/global_mmlu/default/global_mmlu_fr.yaml

-# Generated by _generate_configs.py
-dataset_name: fr
-include: _default_yaml
-task: global_mmlu_fr

lm_eval/tasks/global_mmlu/default/global_mmlu_hi.yaml

-# Generated by _generate_configs.py
-dataset_name: hi
-include: _default_yaml
-task: global_mmlu_hi

lm_eval/tasks/global_mmlu/default/global_mmlu_id.yaml

-# Generated by _generate_configs.py
-dataset_name: id
-include: _default_yaml
-task: global_mmlu_id

lm_eval/tasks/global_mmlu/default/global_mmlu_it.yaml

-# Generated by _generate_configs.py
-dataset_name: it
-include: _default_yaml
-task: global_mmlu_it

lm_eval/tasks/global_mmlu/default/global_mmlu_ja.yaml

-# Generated by _generate_configs.py
-dataset_name: ja
-include: _default_yaml
-task: global_mmlu_ja

lm_eval/tasks/global_mmlu/default/global_mmlu_ko.yaml

-# Generated by _generate_configs.py
-dataset_name: ko
-include: _default_yaml
-task: global_mmlu_ko

lm_eval/tasks/global_mmlu/default/global_mmlu_pt.yaml

-# Generated by _generate_configs.py
-dataset_name: pt
-include: _default_yaml
-task: global_mmlu_pt

lm_eval/tasks/global_mmlu/default/global_mmlu_sw.yaml

-# Generated by _generate_configs.py
-dataset_name: sw
-include: _default_yaml
-task: global_mmlu_sw

lm_eval/tasks/global_mmlu/default/global_mmlu_yo.yaml

-# Generated by _generate_configs.py
-dataset_name: yo
-include: _default_yaml
-task: global_mmlu_yo

lm_eval/tasks/global_mmlu/default/global_mmlu_zh.yaml

-# Generated by _generate_configs.py
-dataset_name: zh
-include: _default_yaml
-task: global_mmlu_zh

lm_eval/tasks/global_mmlu/default/hi/_global_mmlu_hi.yaml

+group: global_mmlu_hi
+task:
+  - global_mmlu_hi_business
+  - global_mmlu_hi_humanities
+  - global_mmlu_hi_medical
+  - global_mmlu_hi_other
+  - global_mmlu_hi_stem
+  - global_mmlu_hi_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/default/hi/_hi_template_yaml

+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: hi
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_business.yaml

+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_hi_business

lm_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_humanities.yaml

+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_hi_humanities

lm_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_medical.yaml

+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_hi_medical

lm_eval/tasks/global_mmlu/default/hi/global_mmlu_hi_other.yaml

+# Generated by _generate_configs.py
+include: _hi_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_hi_other