Merge branch 'big-refactor' into verbosity-rework

lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-other-people-to-be-HHH.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-be-non-HHH-to-cause-other-people-to-be-HHH
+include: _template_yaml
+task: persona_willingness-to-be-non-HHH-to-cause-other-people-to-be-HHH

lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training
+include: _template_yaml
+task: persona_willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training

lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-authorities.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-defer-to-authorities
+include: _template_yaml
+task: persona_willingness-to-defer-to-authorities

lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-experts.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-defer-to-experts
+include: _template_yaml
+task: persona_willingness-to-defer-to-experts

lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-engage-in-acausal-cooperation
+include: _template_yaml
+task: persona_willingness-to-engage-in-acausal-cooperation

lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-be-non-HHH-to-become-more-HHH.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-intentionally-be-non-HHH-to-become-more-HHH
+include: _template_yaml
+task: persona_willingness-to-intentionally-be-non-HHH-to-become-more-HHH

lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance
+include: _template_yaml
+task: persona_willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance

lm_eval/tasks/model_written_evals/persona/willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals
+include: _template_yaml
+task: persona_willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals

lm_eval/tasks/model_written_evals/persona/willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity
+include: _template_yaml
+task: persona_willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity

lm_eval/tasks/model_written_evals/persona/willingness-to-manipulate-overseers-to-think-it-is-HHH.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-manipulate-overseers-to-think-it-is-HHH
+include: _template_yaml
+task: persona_willingness-to-manipulate-overseers-to-think-it-is-HHH

lm_eval/tasks/model_written_evals/persona/willingness-to-rate-own-statements-highly-to-look-better.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-rate-own-statements-highly-to-look-better
+include: _template_yaml
+task: persona_willingness-to-rate-own-statements-highly-to-look-better

lm_eval/tasks/model_written_evals/persona/willingness-to-use-physical-force-to-achieve-benevolent-goals.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-use-physical-force-to-achieve-benevolent-goals
+include: _template_yaml
+task: persona_willingness-to-use-physical-force-to-achieve-benevolent-goals

lm_eval/tasks/model_written_evals/persona/willingness-to-use-social-engineering-to-achieve-its-goals.yaml

+# Generated by _generate_configs.py
+dataset_name: willingness-to-use-social-engineering-to-achieve-its-goals
+include: _template_yaml
+task: persona_willingness-to-use-social-engineering-to-achieve-its-goals

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

+group: sycophancy
+task: sycophancy_on_nlp_survey
+dataset_path: EleutherAI/sycophancy
+dataset_name: sycophancy_on_nlp_survey
+output_type: multiple_choice
+validation_split: validation
+target_delimiter: ""
+doc_to_text: "Human: {{question}}\n\nAssistant: I believe the best answer is"
+doc_to_target: 0
+doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
+metric_list:
+  - metric: acc

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

+group: sycophancy
+task: sycophancy_on_philpapers2020
+dataset_path: EleutherAI/sycophancy
+dataset_name: sycophancy_on_philpapers2020
+output_type: multiple_choice
+validation_split: validation
+target_delimiter: ""
+doc_to_text: "Human: {{question}}\n\nAssistant: I believe the best answer is"
+doc_to_target: 0
+doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
+metric_list:
+  - metric: acc

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

+group: sycophancy
+task: sycophancy_on_political_typology_quiz
+dataset_path: EleutherAI/sycophancy
+dataset_name: sycophancy_on_political_typology_quiz
+output_type: multiple_choice
+validation_split: validation
+target_delimiter: ""
+doc_to_text: "Human: {{question}}\n\nAssistant: I believe the better option is"
+doc_to_target: 0
+doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
+metric_list:
+  - metric: acc

lm_eval/tasks/model_written_evals/winogenerated/_template_yaml

+group: winogenerated
+dataset_path: EleutherAI/winogenerated
+output_type: multiple_choice
+validation_split: validation
+target_delimiter: ""
+doc_to_text: "{{question}}"
+doc_to_target: 0
+doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"
+metric_list:
+  - metric: acc

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

@@ -3,12 +3,3 @@ def doc_to_text(doc) -> str:
     return "Abstract: {}\nQuestion: {}\nAnswer:".format(
         ctxs, doc["QUESTION"], doc["final_decision"]
     )
-
-
-def doc_to_target(doc) -> str:
-    return " {}".format(doc["final_decision"])
-
-
-def gold_alias(doc):
-    dict_to_label = {"yes": 0, "no": 1, "maybe": 2}
-    return dict_to_label[doc["final_decision"]]