generate_configs.py

import yaml
import inspect
import datasets

from importlib import import_module


def main() -> None:

    dataset_path = "persona"
    dataset_full_path = inspect.getfile(
        import_module(f"lm_eval.datasets.model_written_evals.{dataset_path}")
    )
    for task in datasets.get_dataset_infos(dataset_full_path).keys():
        file_name = f"{dataset_path}/{task}.yaml"
        try:
            with open(f"{file_name}", "w") as f:
                f.write("# Generated by generate_configs.py\n")
                yaml.dump(
                    {
                        "include": "template_yaml",
                        "task": f"{dataset_path}_{task}",
                        "dataset_name": task,
                    },
                    f,
                )
        except FileExistsError:
            pass


if __name__ == "__main__":
    main()