Merge pull request #176 from EleutherAI/per_char_agg

Do per character loss aggregation for multiple choice tasks

lm_eval/base.py

@@ -226,19 +226,24 @@ class MultipleChoiceTask(Task):
         gold = doc["gold"]
 
         acc = 1. if np.argmax(results) == gold else 0.
+        completion_len = np.array([float(len(i)) for i in doc["choices"]])
+        acc_norm = 1. if np.argmax(results / completion_len) == gold else 0.
 
         return {
-            "acc": acc
+            "acc": acc,
+            "acc_norm": acc_norm,
         }
     
     def higher_is_better(self):
         return {
-            "acc": True
+            "acc": True,
+            "acc_norm": True,
         }
     
     def aggregation(self):
         return {
-            "acc": mean
+            "acc": mean,
+            "acc_norm": mean,
         }
 
 

lm_eval/tasks/piqa.py

 import numpy as np
-from lm_eval.base import rf
+from lm_eval.base import MultipleChoiceTask, rf
 from ..metrics import mean
 from . common import HFTask
 
 
-class PiQA(HFTask):
+class PiQA(HFTask, MultipleChoiceTask):
     DATASET_PATH = "piqa"
     DATASET_NAME = None
 
@@ -21,29 +21,29 @@ class PiQA(HFTask):
         # TODO: figure out fewshot description
         return ""
 
-    def doc_to_text(self, doc):
-        return "Question: "+doc["goal"] + "\nAnswer:"
+    def _convert_standard(self, doc):
+        out_doc = {
+            "goal": doc["goal"],
+            "choices": [doc["sol1"], doc["sol2"]],
+            "gold": doc["label"],
+        }
+        return out_doc
 
-    def doc_to_target(self, doc):
-        solutions = [doc["sol1"], doc["sol2"]]
-        return " " + solutions[doc["label"]]
+    def _load_docs(self, docs):
+        for record in docs:
+            yield self._convert_standard(record)
 
-    def construct_requests(self, doc, ctx):
-        ll_1, _ = rf.loglikelihood(ctx, " " + doc['sol1'])
-        ll_2, _ = rf.loglikelihood(ctx, " " + doc['sol2'])
-        return ll_1, ll_2
+    def training_docs(self):
+        docs = super().training_docs()
+        return self._load_docs(docs)
 
-    def process_results(self, doc, results):
-        return {
-            'acc': np.argmax(results) == doc["label"]
-        }
+    def validation_docs(self):
+        docs = super().validation_docs()
+        return self._load_docs(docs)
 
-    def aggregation(self):
-        return {
-            'acc': mean
-        }
+    def test_docs(self):
+        docs = super().test_docs()
+        return self._load_docs(docs)
 
-    def higher_is_better(self):
-        return {
-            'acc': True
-        }
+    def doc_to_text(self, doc):
+        return "Question: " + doc["goal"] + "\nAnswer:"