Add WebQs (data only)

lm_eval/tasks/__init__.py

@@ -2,6 +2,7 @@ from . import superglue
 from . import glue
 from . import arc
 from . import race
+from . import webqs
 
 TASK_REGISTRY = {
     "cola": glue.CoLA,
@@ -21,6 +22,7 @@ TASK_REGISTRY = {
     "arc_easy": arc.ARCEasy,
     "arc_challenge": arc.ARCChallenge,
     "race": race.RACE,
+    "webqs": webqs.WebQs,
 }
 
 

lm_eval/tasks/webqs.py

+from . common import HFNLPTask
+
+class WebQs(HFNLPTask):
+    NLP_PATH = "web_questions"
+    NLP_NAME = None
+
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def fewshot_description(self):
+        # TODO: figure out description
+        return ""
+
+    def doc_to_text(self, doc, include_target=True):
+        print(doc)
+        q = "Question: " + doc['question'] + '\n'
+
+        # this picks one answer to be the "correct" one, despite sometimes 
+        # multiple correct answers being possible.
+        # TODO: make sure we're actually handling multi-answer correctly
+        a = "Answer:" + ((" " + doc['answers'][0]) if include_target else '')
+        return q + a
+
+    def evaluate(self, docs, lm, provide_description, num_fewshot):
+        # TODO: implement
+        raise NotImplementedError()
\ No newline at end of file