adjustments

lm_eval/tasks/model_written_evals/generate_configs.py → lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py

@@ -2,24 +2,21 @@ import yaml
 import inspect
 import datasets
 
-from importlib import import_module
+from tqdm import tqdm
 
 
 def main() -> None:
 
-    dataset_path = "persona"
-    dataset_full_path = inspect.getfile(
-        import_module(f"lm_eval.datasets.model_written_evals.{dataset_path}")
-    )
-    for task in datasets.get_dataset_infos(dataset_full_path).keys():
-        file_name = f"{dataset_path}/{task}.yaml"
+    dataset_path = "EleutherAI/advanced_ai_risk"
+    for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()):
+        file_name = f"{task}.yaml"
         try:
             with open(f"{file_name}", "w") as f:
-                f.write("# Generated by generate_configs.py\n")
+                f.write("# Generated by _generate_configs.py\n")
                 yaml.dump(
                     {
-                        "include": "template_yaml",
-                        "task": f"{dataset_path}_{task}",
+                        "include": "_template_yaml",
+                        "task": f"{dataset_path.split('/')[-1]}_{task}",
                         "dataset_name": task,
                     },
                     f,

lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml

-group: advance_ai_risk
-dataset_path: EleutherAI/advance_ai_risk
+group: advanced_ai_risk
+dataset_path: EleutherAI/advanced_ai_risk
 output_type: multiple_choice
 validation_split: train
 doc_to_text: "Human: {question}\n\nAssistant:"

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-coordinate-itself
+include: _template_yaml
+task: advanced_ai_risk_fewshot-coordinate-itself

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-coordinate-other-ais
+include: _template_yaml
+task: advanced_ai_risk_fewshot-coordinate-other-ais

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-coordinate-other-versions
+include: _template_yaml
+task: advanced_ai_risk_fewshot-coordinate-other-versions

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-corrigible-less-HHH
+include: _template_yaml
+task: advanced_ai_risk_fewshot-corrigible-less-HHH

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-corrigible-more-HHH
+include: _template_yaml
+task: advanced_ai_risk_fewshot-corrigible-more-HHH

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-corrigible-neutral-HHH
+include: _template_yaml
+task: advanced_ai_risk_fewshot-corrigible-neutral-HHH

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-myopic-reward
+include: _template_yaml
+task: advanced_ai_risk_fewshot-myopic-reward

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-one-box-tendency
+include: _template_yaml
+task: advanced_ai_risk_fewshot-one-box-tendency

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-power-seeking-inclination
+include: _template_yaml
+task: advanced_ai_risk_fewshot-power-seeking-inclination

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-self-awareness-general-ai
+include: _template_yaml
+task: advanced_ai_risk_fewshot-self-awareness-general-ai

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-self-awareness-good-text-model
+include: _template_yaml
+task: advanced_ai_risk_fewshot-self-awareness-good-text-model

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-self-awareness-text-model
+include: _template_yaml
+task: advanced_ai_risk_fewshot-self-awareness-text-model

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-self-awareness-training-architecture
+include: _template_yaml
+task: advanced_ai_risk_fewshot-self-awareness-training-architecture

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-self-awareness-training-web-gpt
+include: _template_yaml
+task: advanced_ai_risk_fewshot-self-awareness-training-web-gpt

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-survival-instinct
+include: _template_yaml
+task: advanced_ai_risk_fewshot-survival-instinct

lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml

+# Generated by _generate_configs.py
+dataset_name: fewshot-wealth-seeking-inclination
+include: _template_yaml
+task: advanced_ai_risk_fewshot-wealth-seeking-inclination

lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml

+# Generated by _generate_configs.py
+dataset_name: human-coordinate-itself
+include: _template_yaml
+task: advanced_ai_risk_human-coordinate-itself

lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml

+# Generated by _generate_configs.py
+dataset_name: human-coordinate-other-ais
+include: _template_yaml
+task: advanced_ai_risk_human-coordinate-other-ais