Merge branch 'main' into tasklist

# Conflicts:
#	pyproject.toml

lm_eval/tasks/egymmlu/_default_egymmlu_template_yaml

+dataset_path: UBC-NLP/EgyMMLU
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: !function utils.doc_to_text
+doc_to_choice: !function utils.doc_to_choice
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

lm_eval/tasks/egymmlu/_egymmlu.yaml

+group: egymmlu
+group_alias: EgyMMLU
+task:
+- egymmlu_mmlu
+- egymmlu_ar_mmlu
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/egymmlu/_egymmlu_ar_mmlu.yaml

+group: egymmlu_ar_mmlu
+group_alias: ArabicMMLU
+task:
+  - egymmlu_ar_mmlu_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/egymmlu/_egymmlu_mmlu.yaml

+group: egymmlu_mmlu
+group_alias: MMLU
+task:
+  - egymmlu_mmlu_tasks
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0

lm_eval/tasks/egymmlu/_generate_configs.py

+"""
+Take in a YAML, and output all "other" splits with this YAML
+"""
+
+import argparse
+import logging
+import os
+
+import yaml
+from tqdm import tqdm
+
+
+eval_logger = logging.getLogger("lm-eval")
+
+
+MMLU_SUBJECTS = {
+    "global_facts": "other",
+    "high_school_european_history": "humanities",
+    "high_school_geography": "social_sciences",
+    "high_school_government_and_politics": "social_sciences",
+    "high_school_psychology": "social_sciences",
+    "high_school_statistics": "stem",
+    "high_school_world_history": "humanities",
+    "human_aging": "other",
+    "international_law": "humanities",
+    "jurisprudence": "humanities",
+    "logical_fallacies": "humanities",
+    "management": "other",
+    "marketing": "other",
+    "moral_disputes": "humanities",
+    "moral_scenarios": "humanities",
+    "nutrition": "other",
+    "philosophy": "humanities",
+    "professional_law": "humanities",
+    "professional_psychology": "social_sciences",
+    "public_relations": "social_sciences",
+    "security_studies": "social_sciences",
+    "sociology": "social_sciences",
+    "world_religions": "humanities",
+}
+
+
+ARABIC_MMLU_SUBJECTS = {
+    "islamic_studies": "humanities",
+    "driving_test": "other",
+    "natural_science": "stem",
+    "history": "humanities",
+    "general_knowledge": "other",
+    "law": "humanities",
+    "physics": "stem",
+    "social_science": "social_sciences",
+    "management_ar": "other",
+    "arabic_language": "language",
+    "political_science": "social_sciences",
+    "philosophy_ar": "humanities",
+    "accounting": "social_sciences",
+    "computer_science": "stem",
+    "geography": "social_sciences",
+    "math": "stem",
+    "biology": "stem",
+    "economics": "social_sciences",
+    "arabic_language_(general)": "language",
+    "arabic_language_(grammar)": "language",
+    "civics": "social_sciences",
+}
+
+
+DATASETS = {
+    "mmlu": MMLU_SUBJECTS,
+    "ar_mmlu": ARABIC_MMLU_SUBJECTS,
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base_yaml_path", default="_default_egymmlu_template_yaml")
+    parser.add_argument("--save_prefix_path", default="egymmlu")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    base_yaml_name = os.path.split(args.base_yaml_path)[-1]
+
+    ALL_CATEGORIES = []
+    for dataset, SUBJECTS in DATASETS.items():
+        for subject, category in tqdm(SUBJECTS.items()):
+            if category not in ALL_CATEGORIES:
+                ALL_CATEGORIES.append(category)
+
+            yaml_dict = {
+                "include": base_yaml_name,
+                "tag": [
+                    f"egymmlu_{category}_tasks",
+                    "egymmlu_" + dataset + "_tasks",
+                ],
+                "task": f"egymmlu_{subject}",
+                "task_alias": subject.replace("_", " "),
+                "dataset_name": subject,
+            }
+
+            file_save_path = args.save_prefix_path + f"_{subject}.yaml"
+            eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}")
+            with open(file_save_path, "w", encoding="utf-8") as yaml_file:
+                yaml.dump(
+                    yaml_dict,
+                    yaml_file,
+                    allow_unicode=True,
+                    default_style='"',
+                )
+
+    egymmlu_subcategories = [f"egymmlu_{category}" for category in ALL_CATEGORIES]
+
+    file_save_path = args.save_prefix_path + ".yaml"
+
+    eval_logger.info(f"Saving benchmark config to {file_save_path}")

lm_eval/tasks/egymmlu/egymmlu_accounting.yaml

+"dataset_name": "accounting"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_accounting"
+"task_alias": "accounting"

lm_eval/tasks/egymmlu/egymmlu_arabic_language.yaml

+"dataset_name": "arabic_language"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_language_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_arabic_language"
+"task_alias": "arabic language"

lm_eval/tasks/egymmlu/egymmlu_arabic_language_(general).yaml

+"dataset_name": "arabic_language_(general)"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_language_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_arabic_language_(general)"
+"task_alias": "arabic language (general)"

lm_eval/tasks/egymmlu/egymmlu_arabic_language_(grammar).yaml

+"dataset_name": "arabic_language_(grammar)"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_language_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_arabic_language_(grammar)"
+"task_alias": "arabic language (grammar)"

lm_eval/tasks/egymmlu/egymmlu_biology.yaml

+"dataset_name": "biology"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_stem_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_biology"
+"task_alias": "biology"

lm_eval/tasks/egymmlu/egymmlu_civics.yaml

+"dataset_name": "civics"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_civics"
+"task_alias": "civics"

lm_eval/tasks/egymmlu/egymmlu_computer_science.yaml

+"dataset_name": "computer_science"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_stem_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_computer_science"
+"task_alias": "computer science"

lm_eval/tasks/egymmlu/egymmlu_driving_test.yaml

+"dataset_name": "driving_test"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_other_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_driving_test"
+"task_alias": "driving test"

lm_eval/tasks/egymmlu/egymmlu_economics.yaml

+"dataset_name": "economics"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_economics"
+"task_alias": "economics"

lm_eval/tasks/egymmlu/egymmlu_general_knowledge.yaml

+"dataset_name": "general_knowledge"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_other_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_general_knowledge"
+"task_alias": "general knowledge"

lm_eval/tasks/egymmlu/egymmlu_geography.yaml

+"dataset_name": "geography"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_ar_mmlu_tasks"
+"task": "egymmlu_geography"
+"task_alias": "geography"

lm_eval/tasks/egymmlu/egymmlu_global_facts.yaml

+"dataset_name": "global_facts"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_other_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_global_facts"
+"task_alias": "global facts"

lm_eval/tasks/egymmlu/egymmlu_high_school_european_history.yaml

+"dataset_name": "high_school_european_history"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_humanities_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_high_school_european_history"
+"task_alias": "high school european history"

lm_eval/tasks/egymmlu/egymmlu_high_school_geography.yaml

+"dataset_name": "high_school_geography"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_high_school_geography"
+"task_alias": "high school geography"

lm_eval/tasks/egymmlu/egymmlu_high_school_government_and_politics.yaml

+"dataset_name": "high_school_government_and_politics"
+"include": "_default_egymmlu_template_yaml"
+"tag":
+- "egymmlu_social_sciences_tasks"
+- "egymmlu_mmlu_tasks"
+"task": "egymmlu_high_school_government_and_politics"
+"task_alias": "high school government and politics"