fixes and adjustments

lm_eval/api/task.py

@@ -687,7 +687,7 @@ class ConfigurableTask(Task):
             for choice in check_choices:
                 choice_has_whitespace = True if choice[0].isspace() else False
                 delimiter_has_whitespace = (
-                    True if self.config.target_delimiter[-1].isspace() else False
+                    True if (len(self.config.target_delimiter) >= 1 and self.config.target_delimiter[-1].isspace()) else False
                 )
 
                 if delimiter_has_whitespace and choice_has_whitespace:

lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml

 group: advanced_ai_risk
 dataset_path: EleutherAI/advanced_ai_risk
 output_type: multiple_choice
-validation_split: train
+validation_split: validation
+target_delimiter: ""
 doc_to_text: "Human: {question}\n\nAssistant:"
 doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"

lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-web-gpt.yaml → lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml

 # Generated by _generate_configs.py
-dataset_name: human-self-awareness-training-web-gpt
+dataset_name: human-self-awareness-web-gpt
 include: _template_yaml
-task: advanced_ai_risk_human-self-awareness-training-web-gpt
+task: advanced_ai_risk_human-self-awareness-web-gpt

lm_eval/tasks/model_written_evals/persona/_template_yaml

@@ -2,6 +2,7 @@ group: persona
 dataset_path: EleutherAI/persona
 output_type: multiple_choice
 validation_split: validation
+target_delimiter: ""
 doc_to_text: "{{question}}"
 doc_to_target: 0
 doc_to_choice: "{{[answer_matching_behavior, answer_not_matching_behavior]}}"

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

 group: sycophancy
+task: sycophancy_on_nlp_survey
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_nlp_survey
 output_type: multiple_choice

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

 group: sycophancy
+task: sycophancy_on_philpapers2020
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_philpapers2020
 output_type: multiple_choice

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

 group: sycophancy
+task: sycophancy_on_political_typology_quiz
 dataset_path: EleutherAI/sycophancy
 dataset_name: sycophancy_on_political_typology_quiz
 output_type: multiple_choice