_generate_configs.py

"""
Take in a YAML, and output all "other" splits with this YAML
"""

import argparse
import logging
import os

import yaml
from tqdm import tqdm


eval_logger = logging.getLogger("lm-eval")


SUBJECTS = {'Islamic Studies': 'humanities',
 'Driving Test': 'other',
 'Natural Science (Middle School)': 'stem',
 'Natural Science (Primary School)': 'stem',
 'History (Primary School)': 'humanities',
 'History (Middle School)': 'humanities',
 'History (High School)': 'humanities',
 'General Knowledge': 'other',
 'General Knowledge (Primary School)': 'other',
 'General Knowledge (Middle School)': 'other',
 'Law (Professional)': 'humanities',
 'Physics (High School)': 'stem',
 'Social Science (Middle School)': 'social_science',
 'Social Science (Primary School)': 'social_science',
 'Management (University)': 'other',
 'Arabic Language (Primary School)': 'language',
 'Arabic Language (Middle School)': 'language',
 'Arabic Language (High School)': 'language',
 'Political Science (University)': 'social_science',
 'Philosophy (High School)': 'humanities',
 'Accounting (University)': 'social_science',
 'Computer Science (University)': 'stem',
 'Computer Science (Middle School)': 'stem',
 'Computer Science (Primary School)': 'stem',
 'Computer Science (High School)': 'stem',
 'Geography (Primary School)': 'social_science',
 'Geography (Middle School)': 'social_science',
 'Geography (High School)': 'social_science',
 'Math (Primary School)': 'stem',
 'Biology (High School)': 'stem',
 'Economics (University)': 'social_science',
 'Economics (Middle School)': 'social_science',
 'Economics (High School)': 'social_science',
 'Arabic Language (General)': 'language',
 'Arabic Language (Grammar)': 'language',
 'Islamic Studies (High School)': 'humanities',
 'Islamic Studies (Middle School)': 'humanities',
 'Islamic Studies (Primary School)': 'humanities',
 'Civics (Middle School)': 'social_science',
 'Civics (High School)': 'social_science'}


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--base_yaml_path", default="_default_arabicmmlu_template_yaml")
    parser.add_argument("--save_prefix_path", default="arabicmmlu")
    return parser.parse_args()


if __name__ == "__main__":
    args = parse_args()

    # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
    base_yaml_name = os.path.split(args.base_yaml_path)[-1]

    # with open(args.base_yaml_path, encoding="utf-8") as f:
    #     base_yaml = yaml.full_load(f)

    ALL_CATEGORIES = []
    for subject, category in tqdm(SUBJECTS.items()):
        if category not in ALL_CATEGORIES:
            ALL_CATEGORIES.append(category)

        # description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"

        yaml_dict = {
            "include": base_yaml_name,
            "tag": f"arabicmmlu_{category}_tasks",
            "task": f"arabicmmlu_{subject.lower().replace(' ', '_').replace('(', '').replace(')', '')}",
            "task_alias": subject,
            "dataset_name": subject,
            # "description": description,
        }

        file_save_path = (
            args.save_prefix_path
            + f"_{subject.lower().replace(' ', '_').replace('(', '').replace(')', '')}.yaml"
        )
        eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}")
        with open(file_save_path, "w", encoding="utf-8") as yaml_file:
            yaml.dump(
                yaml_dict,
                yaml_file,
                allow_unicode=True,
                default_style='"',
            )

    arabicmmlu_subcategories = [f"arabicmmlu_{category}" for category in ALL_CATEGORIES]

    file_save_path = args.save_prefix_path + ".yaml"

    eval_logger.info(f"Saving benchmark config to {file_save_path}")
    with open(file_save_path, "w", encoding="utf-8") as yaml_file:
        yaml.dump(
            {
                "group": "arabicmmlu",
                "task": arabicmmlu_subcategories,
            },
            yaml_file,
            indent=4,
            default_flow_style=False,
        )