fix doc_to_target to return int rather than str (deprecated)

lm_eval/api/task.py

@@ -971,10 +971,6 @@ class ConfigurableTask(Task):
                 )
                 test_target = test_choice[test_target]
 
-            assert hasattr(test_choice, "__iter__") and not isinstance(
-                test_choice, (str, bytes)
-            ), f"[{self.config.task}] doc_to_choice must be an iterable!"
-
             for choice in test_choice:
                 choice_has_whitespace = choice[0].isspace()
                 delimiter_has_whitespace = (

lm_eval/tasks/super_glue/record/util.py

@@ -19,7 +19,7 @@ def format_answer(query, entity):
 
 def doc_to_target(doc):
     # We only output the first correct entity in a doc
-    return format_answer(query=doc["query"], entity=doc["answers"][0])
+    return doc["entities"].index(doc["answers"][0])
 
 
 def doc_to_choice(doc):

tests/test_tasks.py

@@ -93,7 +93,7 @@ class BaseTasks:
         _array = [task.doc_to_text(doc) for doc in arr]
         # space convention; allow txt to have length 0 for perplexity-like tasks since the model tacks an <|endoftext|> on
         target_delimiter: str = task.config.target_delimiter
-        if not task.multiple_input:
+        if not task.multiple_inputs:
             for x in _array:
                 assert isinstance(x, str)
                 assert (
@@ -148,7 +148,7 @@ class BaseTasks:
         # ctx is "" for multiple input tasks
         requests = [
             task.construct_requests(
-                doc=doc, ctx="" if task.multiple_input else task.doc_to_text(doc)
+                doc=doc, ctx="" if task.multiple_inputs else task.doc_to_text(doc)
             )
             for doc in arr
         ]