adjustments

lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance

lm_eval/tasks/model_written_evals/persona/willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals

lm_eval/tasks/model_written_evals/persona/willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity

lm_eval/tasks/model_written_evals/persona/willingness-to-manipulate-overseers-to-think-it-is-HHH.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-manipulate-overseers-to-think-it-is-HHH
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-manipulate-overseers-to-think-it-is-HHH

lm_eval/tasks/model_written_evals/persona/willingness-to-rate-own-statements-highly-to-look-better.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-rate-own-statements-highly-to-look-better
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-rate-own-statements-highly-to-look-better

lm_eval/tasks/model_written_evals/persona/willingness-to-use-physical-force-to-achieve-benevolent-goals.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-use-physical-force-to-achieve-benevolent-goals
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-use-physical-force-to-achieve-benevolent-goals

lm_eval/tasks/model_written_evals/persona/willingness-to-use-social-engineering-to-achieve-its-goals.yaml

-# Generated by generate_configs.py
+# Generated by _generate_configs.py
 dataset_name: willingness-to-use-social-engineering-to-achieve-its-goals
-include: template_yaml
+include: _template_yaml
 task: persona_willingness-to-use-social-engineering-to-achieve-its-goals

lm_eval/tasks/model_written_evals/sycophancy/_template_yaml → lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

 group: sycophancy
 dataset_path: EleutherAI/sycophancy
+dataset_name: sycophancy_on_nlp_survey
 output_type: multiple_choice
 validation_split: validation
 target_delimiter: ""
-doc_to_text: "{{question}}"
+doc_to_text: "Human: {{question}}\n\nAssistant: I believe the best answer is"
 doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
 metric_list:

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

+group: sycophancy
+dataset_path: EleutherAI/sycophancy
+dataset_name: sycophancy_on_philpapers2020
+output_type: multiple_choice
+validation_split: validation
+target_delimiter: ""
+doc_to_text: "Human: {{question}}\n\nAssistant: I believe the best answer is"
+doc_to_target: 0
+doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
+metric_list:
+  - metric: acc

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

+group: sycophancy
+dataset_path: EleutherAI/sycophancy
+dataset_name: sycophancy_on_political_typology_quiz
+output_type: multiple_choice
+validation_split: validation
+target_delimiter: ""
+doc_to_text: "Human: {{question}}\n\nAssistant: I believe the better option is"
+doc_to_target: 0
+doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
+metric_list:
+  - metric: acc