Implement unit testing and fix lots of problems with tasks

lm_eval/models/dummy.py

-# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
-
+import random
 from lm_eval.base import LM
-from . import MODEL_REGISTRY
 
 
-@MODEL_REGISTRY.register("dummy")
 class DummyLM(LM):
+    def __init__(self):
+        pass
+
+    @classmethod
+    def create_from_arg_string(cls, arg_string):
+        return cls()
+
+    def loglikelihood(self, requests):
+        res = []
+        
+        for _ in requests:
+            res.append((-random.random(), False))
+
+        return res
     
-    def loglikelihood(self, context, continuation):
-        return 0.0
+    def greedy_until(self, requests):
+        # TODO: implement
+        pass
\ No newline at end of file

lm_eval/models/gpt2.py

@@ -19,7 +19,6 @@ class GPT2LM(LM):
         return cls(device=args.get("device", "cpu"))
 
     def loglikelihood(self, requests):
-        print(requests)
         res = []
         # TODO: vectorize properly
         for context, continuation in tqdm(requests):

lm_eval/models/gpt3.py

@@ -32,6 +32,8 @@ class GPT3LM(LM):
         return cls(engine=args.get("engine", "davinci"))
 
     def loglikelihood(self, context, continuation):
+        # TODO: implement new framework
+        
         import openai
         
         context_enc = self.tokenizer.encode(context)

lm_eval/tasks/__init__.py

@@ -23,7 +23,7 @@ TASK_REGISTRY = {
     "rte": glue.RTE,
     "qnli": glue.QNLI,
     "qqp": glue.QQP,
-    "stsb": glue.STSB,
+    #"stsb": glue.STSB, # not implemented yet
     "sst": glue.SST,
     "wnli": glue.WNLI,
     # SuperGLUE
@@ -33,23 +33,25 @@ TASK_REGISTRY = {
     "multirc": superglue.MultiRC,
     "record": superglue.ReCoRD,
     "wic": superglue.WordsInContext,
-    "wsc": superglue.SGWinogradSchemaChallenge,
+    #"wsc": superglue.SGWinogradSchemaChallenge, # not implemented yet
+    
     # Order by benchmark/genre?
-    "arc_easy": arc.ARCEasy,
-    "arc_challenge": arc.ARCChallenge,
-    "quac": quac.QuAC,
-    "hellaswag": hellaswag.HellaSwag,
-    "openbookqa": openbookqa.OpenBookQA,
-    "sat": sat.SATAnalogies,
-    "squad": squad.SQuAD,
-    "race": race.RACE,
-    "naturalqs": naturalqs.NaturalQs,
-    "webqs": webqs.WebQs,
-    "wsc273": wsc273.WinogradSchemaChallenge273,
-    "winogrande": winogrande.Winogrande,
-    "anli_r1": anli.ANLIRound1,
-    "anli_r2": anli.ANLIRound2,
-    "anli_r3": anli.ANLIRound3,
+
+    # "arc_easy": arc.ARCEasy, # not implemented yet
+    # "arc_challenge": arc.ARCChallenge, # not implemented yet
+    # "quac": quac.QuAC, # not implemented yet
+    # "hellaswag": hellaswag.HellaSwag, # not implemented yet
+    # "openbookqa": openbookqa.OpenBookQA, # not implemented yet
+    # "sat": sat.SATAnalogies, # not implemented yet
+    # "squad": squad.SQuAD, # not implemented yet
+    # "race": race.RACE, # not implemented yet
+    # "naturalqs": naturalqs.NaturalQs, # not implemented yet
+    # "webqs": webqs.WebQs, # not implemented yet
+    # "wsc273": wsc273.WinogradSchemaChallenge273, # not implemented yet
+    # "winogrande": winogrande.Winogrande, # not implemented yet
+    # "anli_r1": anli.ANLIRound1, # not implemented yet
+    # "anli_r2": anli.ANLIRound2, # not implemented yet
+    # "anli_r3": anli.ANLIRound3, # not implemented yet
     # arithmetic
     "arithmetic_2da": arithmetic.Arithmetic2DPlus,
     "arithmetic_2ds": arithmetic.Arithmetic2DMinus,

lm_eval/tasks/arithmetic.py

@@ -12,7 +12,6 @@ class Arithmetic(Dataset):
 
     def __init__(self):
         super().__init__()
-        self.set_docs()
 
     def download(self):
         file_name, checksum = self.get_file_download_info()
@@ -20,6 +19,7 @@ class Arithmetic(Dataset):
         if not os.path.exists(self.directory):
             os.makedirs(self.directory)
         download_file(url, self.directory+file_name, checksum)
+        self.set_docs()
 
     @abc.abstractmethod
     def get_file_download_info(self):

lm_eval/tasks/common.py

@@ -11,6 +11,8 @@ class HFTask(Dataset):
     def __init__(self):
         super().__init__()
         self._training_docs = None
+    
+    def download(self):
         self.data = datasets.load_dataset(path=self.DATASET_PATH, name=self.DATASET_NAME)
 
     def has_training_docs(self):

lm_eval/tasks/drop.py

@@ -11,7 +11,7 @@ class DROP(Dataset):
     DATAFOLDER = Path(__file__).parent / "../../data/drop"
     
     def __init__(self):
-        self.download()
+        super().__init__()
 
     def has_training_docs(self):
         """Whether the task has a training set"""

lm_eval/tasks/race.py

@@ -54,16 +54,13 @@ class RACE(HFTask):
         # TODO: figure out description
         return ""
 
-    def doc_to_text(self, doc, include_target=True):
-        r = "Article:\n" + doc['article'] + '\n\n'
+    def doc_to_text(self, doc):
+        # TODO: implement
+        pass
 
-        r += doc['problems'] >> apply(enumerate) >> each(
-            lambda x: 'Q: ' + x[1]['question'] + '\n\nA:' 
-            + ((' ' + x[1]['options'][['A', 'B', 'C', 'D'].index(x[1]['answer'])]) \
-                if x[0] != len(doc['problems']) - 1 or include_target else '')) \
-            >> join('\n\n')
-
-        return r
+    def doc_to_target(self, doc):
+        # TODO: implement
+        pass
 
     def construct_requests(self, doc, ctx):
         """ Uses RequestFactory to construct Requests and returns an iterable of 

lm_eval/tasks/sat.py

@@ -9,6 +9,8 @@ from ..utils import sh
 
 
 class SATAnalogies(Dataset):    
+    NEEDS_MANUAL_DL = True
+    
     def __init__(self):
         super().__init__()
 

lm_eval/tasks/storycloze.py

@@ -5,8 +5,8 @@ from ..utils import sh
 import csv
 
 class StoryCloze(Dataset):
-    def __init__(self):
-        self.download()
+    NEEDS_MANUAL_DL = True
+
     def download(self):
         #TODO: replace with Eye link
         pass

lm_eval/tasks/superglue.py

@@ -75,6 +75,7 @@ class CommitmentBank(HFTask):
         return True
 
     def fewshot_description(self):
+        # TODO: figure out actual description
         return "Given a premise and a hypothesis, classify whether the author of the premise is committed" \
             "to the truth of the hypothesis. The three possible labels are true, false or neither."
 
@@ -145,6 +146,7 @@ class Copa(HFTask):
         return True
 
     def fewshot_description(self):
+        # TODO: figure out actual description
         return "Given a premise and one alternative with a causal relation to the premise and another without," \
             "choose the more plausible alternative"
 
@@ -208,6 +210,7 @@ class MultiRC(HFTask):
         return True
 
     def fewshot_description(self):
+        # TODO: figure out actual description
         return "READING COMPREHENSION ANSWER KEY"
 
     def doc_to_text(self, doc):
@@ -260,13 +263,16 @@ class ReCoRD(HFTask):
     def has_test_docs(self):
         return True
 
+    def fewshot_description(self):
+        # TODO: figure out actual description
+        return ""
+
     def training_docs(self):
         # In ReCoRD, each doc manifests multiple "examples" in the context of few shot example packing.
         # Each doc consists of multiple answer candidates, each of which is scored yes/no.
         # Hence, we one "doc" for each (context + passage, answer) pair.
         # Moreover, we only use the correct answers for context packing
         # (This is not an issue for evaluation, where we can directly score multiple candidates at once).
-        if self.has_training_docs():
         if self._training_docs is None:
             self._training_docs = []
             for doc in self.data["train"]:
@@ -279,6 +285,16 @@ class ReCoRD(HFTask):
                     })
         return self._training_docs
 
+    def validation_docs(self):
+        for doc in self.data["validation"]:
+            for entity in list(set(doc["entities"])):
+                yield {
+                    "passage": doc["passage"],
+                    "query": doc["query"],
+                    "entity": entity,
+                    "label": entity in doc["answers"],
+                }
+
     def doc_to_text(self, doc):
         initial_text, *highlights = doc["passage"].strip().split("\n@highlight\n")
         text = initial_text + "\n\n"
@@ -296,7 +312,7 @@ class ReCoRD(HFTask):
     def construct_requests(self, doc, ctx):
         requests = [
             rf.loglikelihood(ctx, self.format_answer(query=doc["query"], entity=entity))
-            for entity in doc["entities"]
+            for entity in doc["entity"]
         ]
         return requests
 
@@ -342,6 +358,10 @@ class WordsInContext(HFTask):
     def has_test_docs(self):
         return True
 
+    def fewshot_description(self):
+        # TODO: figure out actual description
+        return ""
+
     def doc_to_text(self, doc):
         return "{}\n{}\nQuestion: Is the word '{}' used in the same way in the" \
                " two sentences above?\nanswer:".format(
@@ -405,6 +425,7 @@ class SGWinogradSchemaChallenge(HFTask):
             return self._training_docs
 
     def fewshot_description(self):
+        # TODO: figure out actual description
         return "Final Exam with Answer Key\n" \
            "Instructions: Please carefully read the following passages. " \
            "For each passage, you must identify which noun the pronoun marked in *bold*" \

lm_eval/tasks/triviaqa.py

+import os
 import json
 import random
 from lm_eval.base import Dataset
 from ..utils import sh
 
 class TriviaQA(Dataset):
-    def __init__(self):
-        self.download()
     def download(self):
-        #pass
-        #TODO: don't download if files already there
+        if not os.path.exists('data/triviaqa'):
             sh("""
             mkdir -p data/triviaqa
             wget http://nlp.cs.washington.edu/triviaqa/data/triviaqa-unfiltered.tar.gz -O data/triviaqa/trivia_qa-unfiltered.tar.gz

lm_eval/tasks/wsc273.py

@@ -72,11 +72,13 @@ class WinogradSchemaChallenge273(Dataset):
  
         return docs
 
-    def doc_to_text(self, doc, include_target=True):
-        # WSC273 is currently only writing out full examples. Partial evaluation needs implementing.
-        text = doc['completions']['T'] + ' True. ' + doc['completions']['F'] + ' False.'
-        return text
+    def doc_to_text(self, doc):
+        # TODO: implement
+        pass
 
+    def doc_to_target(self, doc):
+        # TODO: implement
+        pass
 
     def construct_requests(self, doc, ctx):
         """ Uses RequestFactory to construct Requests and returns an iterable of