also apply normalization to piqa

2522c27f · Ben Wang · 3eaa493e · 2522c27f
Commit 2522c27f authored Apr 11, 2021 by Ben Wang
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 1 deletion

lm_eval/tasks/piqa.py lm_eval/tasks/piqa.py +4 -1

No files found.
--- a/lm_eval/tasks/piqa.py
+++ b/lm_eval/tasks/piqa.py
@@ -34,8 +34,11 @@ class PiQA(HFTask):
        return ll_1, ll_2

    def process_results(self, doc, results):
+        completion_len = np.array([float(len(doc["sol1"])), float(len(doc["sol2"]))])
+
        return {
-            'acc': np.argmax(results) == doc["label"]
+            'acc': np.argmax(results) == doc["label"],
+            'acc_norm': np.argmax(results / completion_len) == doc["label"]
        }

    def aggregation(self):