go through choices and targets

lm_eval/api/task.py

@@ -649,20 +649,35 @@ class ConfigurableTask(Task):
 
             if type(test_text) is int:
                 self.multiple_input = num_choice
+        else:
+            test_choice = None
 
         if type(test_target) is list:
             self.multiple_target = len(test_target)
         else:
             if type(test_target) is int:
-                test_target = self.doc_to_choice(test_target)[test_target]
+                test_target = [self.doc_to_choice(test_target)[test_target]]
+            else:
+                test_target = [test_target]
 
-            if (" " in self._config.target_delimiter) and (" " in test_target):
-                eval_logger.warning("Both target_delimiter and target has whitespace")
-            elif (" " not in self._config.target_delimiter) and (
-                " " not in test_target
-            ):
+        if test_choice is not None:
+            check_choices = test_choice
+        else:
+            check_choices = test_target
+
+        for choice in check_choices:
+            choice_has_whitespace = True if " " in choice else False
+            delimiter_has_whitespace = (
+                True if " " in self._config.target_delimiter else False
+            )
+
+            if delimiter_has_whitespace and choice_has_whitespace:
+                eval_logger.warning(
+                    f'Both target_delimiter and target choice: "{choice}" have whitespace'
+                )
+            elif (not delimiter_has_whitespace) and (not choice_has_whitespace):
                 eval_logger.warning(
-                    "Both target_delimiter and target does not have whitespace, ignore if the language you are evaluating on does not require/use whitespace"
+                    f'Both target_delimiter and target choice: "{choice}" does not have whitespace, ignore if the language you are evaluating on does not require/use whitespace'
                 )
 
     def download(self, dataset_kwargs=None):
@@ -1025,7 +1040,6 @@ class ConfigurableTask(Task):
                         predictions=[result],
                         **self._metric_fn_kwargs[key],
                     )
-                    print("score", result)
                 else:
                     # in the case where we have multiple targets,
                     # return true if any are true