add evaluation for TriviaQA dataset based on loglikelihood

lm_eval/tasks/__init__.py

@@ -15,6 +15,7 @@ from . import sat
 from . import arithmetic
 from . import lambada
 from . import piqa
+from . import triviaqa
 
 TASK_REGISTRY = {
     # GLUE
@@ -42,6 +43,7 @@ TASK_REGISTRY = {
     "lambada": lambada.LAMBADA,
     "piqa": piqa.PiQA,
 
+    "triviaqa": triviaqa.TriviaQA,
     # "arc_easy": arc.ARCEasy, # not implemented yet
     # "arc_challenge": arc.ARCChallenge, # not implemented yet
     # "quac": quac.QuAC, # not implemented yet

lm_eval/tasks/lambada.py

@@ -2,7 +2,6 @@ from lm_eval.base import Dataset, rf, mean
 from lm_eval.utils import sh
 import json
 import requests
-import ftfy
 import math
 from best_download import download_file
 

lm_eval/tasks/triviaqa.py

 import os
 import json
 import random
-from lm_eval.base import Dataset
+from lm_eval.base import Dataset, mean, rf
 from ..utils import sh
 
 class TriviaQA(Dataset):
@@ -37,52 +37,27 @@ class TriviaQA(Dataset):
         return ""
     
     def doc_to_text(self, doc):
-        return ''.join(['Q: ', doc['Question'], '\n\n','A: '])
+        return ''.join(['Q:', doc['Question'], '\n\n','A:'])
 
     def doc_to_target(self, doc):
         return doc['Answer']['Aliases'][0]
 
     def construct_requests(self, doc, ctx):
-        """ Uses RequestFactory to construct Requests and returns an iterable of 
-        Requests which will be sent to the LM.
+        ll, is_prediction = rf.loglikelihood(ctx,doc['Answer']['Value'])
+        return is_prediction
 
-        :param doc:
-            The document as returned from training_docs, validation_docs, or test_docs.
-        :param ctx: str
-            The context string, generated by fewshot_context. This includes the natural 
-            language description, as well as the few shot examples, and the question
-            part of the document for `doc`. 
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
-    
     def process_results(self, doc, results):
-        """Take a single document and the LM results and evaluates, returning a 
-        dict where keys are the names of submetrics and values are the values of 
-        the metric for that one document
-
-        :param doc:
-            The document as returned from training_docs, validation_docs, or test_docs.
-        :param results:
-            The results of the requests created in construct_requests.
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
+        is_prediction = results
+        return {
+            "acc": float(is_prediction[1])
+        }
 
     def aggregation(self):
-        """
-        :returns: {str: [float] -> float}
-            A dictionary where keys are the names of submetrics and values are 
-            functions that aggregate a list of metrics
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
+        return {
+            "acc": mean,
+        }
 
     def higher_is_better(self):
-        """
-        :returns: {str: bool}
-            A dictionary where keys are the names of submetrics and values are 
-            whether a higher value of the submetric is better
+        return {
+            "acc": True
+        }
\ No newline at end of file
-        """
-        # TODO: implement evaluation.
-        raise NotImplementedError('Evaluation not implemented')
\ No newline at end of file

lm_eval/utils_stream.py

 import os
 from functools import reduce
 import operator
-import lm_dataformat as lmd
 from tqdm import tqdm
 import json