Merge branch 'main' into llama

lm_eval/tasks/global_mmlu/default/it/utils.py

+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm_eval/tasks/global_mmlu/default/ja/_global_mmlu_ja.yaml

+group: global_mmlu_ja
+task:
+  - global_mmlu_ja_business
+  - global_mmlu_ja_humanities
+  - global_mmlu_ja_medical
+  - global_mmlu_ja_other
+  - global_mmlu_ja_stem
+  - global_mmlu_ja_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/default/ja/_ja_template_yaml

+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: ja
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_business.yaml

+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_ja_business

lm_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_humanities.yaml

+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_ja_humanities

lm_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_medical.yaml

+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_ja_medical

lm_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_other.yaml

+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_ja_other

lm_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_social_sciences.yaml

+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_ja_social_sciences

lm_eval/tasks/global_mmlu/default/ja/global_mmlu_ja_stem.yaml

+# Generated by _generate_configs.py
+include: _ja_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_ja_stem

lm_eval/tasks/global_mmlu/default/ja/utils.py

+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm_eval/tasks/global_mmlu/default/ko/_global_mmlu_ko.yaml

+group: global_mmlu_ko
+task:
+  - global_mmlu_ko_business
+  - global_mmlu_ko_humanities
+  - global_mmlu_ko_medical
+  - global_mmlu_ko_other
+  - global_mmlu_ko_stem
+  - global_mmlu_ko_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/default/ko/_ko_template_yaml

+dataset_path: CohereForAI/Global-MMLU-Lite
+dataset_name: ko
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: default
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_business.yaml

+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_business
+task: global_mmlu_ko_business

lm_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_humanities.yaml

+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_humanities
+task: global_mmlu_ko_humanities

lm_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_medical.yaml

+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_medical
+task: global_mmlu_ko_medical

lm_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_other.yaml

+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_other
+task: global_mmlu_ko_other

lm_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_social_sciences.yaml

+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_social_sciences
+task: global_mmlu_ko_social_sciences

lm_eval/tasks/global_mmlu/default/ko/global_mmlu_ko_stem.yaml

+# Generated by _generate_configs.py
+include: _ko_template_yaml
+process_docs: !function utils.process_stem
+task: global_mmlu_ko_stem

lm_eval/tasks/global_mmlu/default/ko/utils.py

+from functools import partial
+
+
+CATEGORIES = ["Business", "Humanities", "Medical", "Other", "STEM", "Social Sciences"]
+
+
+def process_docs(dataset, category):
+    return dataset.filter(lambda x: x["subject_category"] == category)
+
+
+process_functions = {
+    f"process_{category.lower().replace(' ', '_')}": partial(
+        process_docs, category=category
+    )
+    for category in CATEGORIES
+}
+
+globals().update(process_functions)

lm_eval/tasks/global_mmlu/default/pt/_global_mmlu_pt.yaml

+group: global_mmlu_pt
+task:
+  - global_mmlu_pt_business
+  - global_mmlu_pt_humanities
+  - global_mmlu_pt_medical
+  - global_mmlu_pt_other
+  - global_mmlu_pt_stem
+  - global_mmlu_pt_social_sciences
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0