Fixes to make greedy_until work

# Conflicts:
#	lm_eval/models/gpt2.py
#	lm_eval/tasks/squad.py

lm_eval/base.py

@@ -269,6 +269,7 @@ def perplexity(items):
 
 req_ret_lens = {
     'loglikelihood': 2,
+    'greedy_until': None,
 }
 
 import os
@@ -335,11 +336,15 @@ class Request:
         self.index = index
     
     def __iter__(self):
+        if req_ret_lens[self.type] is None:
+            raise IndexError('This request type does not return multiple arguments!')
         i = 0
         for i in range(req_ret_lens[self.type]):
             yield Request(self.type, self.args, i)
     
     def __getitem__(self, i):
+        if req_ret_lens[self.type] is None:
+            raise IndexError('This request type does not return multiple arguments!')
         return Request(self.type, self.args, i)
     
     def __eq__(self, other):

lm_eval/evaluator.py

@@ -39,6 +39,7 @@ def evaluate(lm, task_dict, provide_description, num_fewshot, limit):
             )
 
             reqs = task.construct_requests(doc, ctx)
+            if not isinstance(reqs, (list, tuple)): reqs = [reqs]
 
             for i, req in enumerate(reqs):
                 requests[req.type].append(req)

lm_eval/models/dummy.py

@@ -19,5 +19,9 @@ class DummyLM(LM):
         return res
     
     def greedy_until(self, requests):
-        # TODO: implement
-        pass
+        res = []
+        
+        for _ in requests:
+            res.append("lol")
+
+        return res

lm_eval/models/gpt2.py

@@ -66,7 +66,7 @@ class GPT2LM(LM):
 
             cont = self.gpt2.generate(
                 context_enc,
-                max_length=self.MAX_GEN_TOKS,
+                max_length=context_enc.shape[1] + self.MAX_GEN_TOKS,
                 eos_token_id=primary_until,
                 do_sample=False
             )

lm_eval/tasks/arithmetic.py

@@ -66,7 +66,7 @@ class Arithmetic(Task):
         return is_prediction
 
     def process_results(self, doc, results):
-        ll, is_prediction = results
+        is_prediction, = results
         return {
             "acc": is_prediction
         }

lm_eval/tasks/squad.py

@@ -30,7 +30,7 @@ class SQuAD(HFTask):
         return "Title: The_Title_of_It\n\nBackground: A text passage as background to answer the question with.\n\nQ: Question about the passage.\n\nA: Answer."
 
     def doc_to_text(self, doc):
-        return 'Title: ' + doc['title'] + '\n\n' + 'Background: ' + doc['context'] + '\n\n' + 'Q: ' + doc['question'] + '\n\n' + 'A: '
+        return 'Title: ' + doc['title'] + '\n\n' + 'Background: ' + doc['context'] + '\n\n' + 'Question: ' + doc['question'] + '\n\n' + 'Answer:'
 
     def doc_to_target(self, doc):
         answer_list = doc['answers']['text']