Merge pull request #781 from EleutherAI/process-whitespace

[Refactor] Process Whitespace for greedy_until

lm_eval/api/task.py

@@ -652,9 +652,36 @@ class ConfigurableTask(Task):
 
             if type(test_text) is int:
                 self.multiple_input = num_choice
+        else:
+            test_choice = None
 
         if type(test_target) is list:
             self.multiple_target = len(test_target)
+        else:
+            if (type(test_target) is int) and (test_choice is not None):
+                test_target = [self.doc_to_choice(test_target)[test_target]]
+            else:
+                test_target = [test_target]
+
+        if test_choice is not None:
+            check_choices = test_choice
+        else:
+            check_choices = test_target
+
+        for choice in check_choices:
+            choice_has_whitespace = True if " " in choice else False
+            delimiter_has_whitespace = (
+                True if " " in self._config.target_delimiter else False
+            )
+
+            if delimiter_has_whitespace and choice_has_whitespace:
+                eval_logger.warning(
+                    f'Both target_delimiter and target choice: "{choice}" have whitespace'
+                )
+            elif (not delimiter_has_whitespace) and (not choice_has_whitespace):
+                eval_logger.warning(
+                    f'Both target_delimiter and target choice: "{choice}" does not have whitespace, ignore if the language you are evaluating on does not require/use whitespace'
+                )
 
     def download(self, dataset_kwargs=None):
 

lm_eval/tasks/super_glue/boolq/seq2seq.yaml

@@ -7,8 +7,8 @@ output_type: greedy_until
 training_split: train
 validation_split: validation
 doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
-doc_to_target: "{{[' no', ' yes'][label]}}"
-target_delimiter: ""
+doc_to_target: label
+doc_to_choice: ['no', 'yes']
 generation_kwargs:
   until:
     - "\n\n"