import yaml import inspect import datasets from importlib import import_module def main() -> None: dataset_path = "persona" dataset_full_path = inspect.getfile( import_module(f"lm_eval.datasets.model_written_evals.{dataset_path}") ) for task in datasets.get_dataset_infos(dataset_full_path).keys(): file_name = f"{dataset_path}/{task}.yaml" try: with open(f"{file_name}", "w") as f: f.write("# Generated by generate_configs.py\n") yaml.dump( { "include": "template_yaml", "task": f"{dataset_path}_{task}", "dataset_name": task, }, f, ) except FileExistsError: pass if __name__ == "__main__": main()