Fix eval

2aed1256 · Niklas Muennighoff · 5ef6ea82 · 2aed1256
Commit 2aed1256 authored Feb 12, 2023 by Niklas Muennighoff
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

lm_eval/tasks/babi.py lm_eval/tasks/babi.py +1 -1

No files found.
--- a/lm_eval/tasks/babi.py
+++ b/lm_eval/tasks/babi.py
@@ -74,7 +74,7 @@ class Babi(Task):
            The results of the requests created in construct_requests.
        """
        gold = doc["answer"]
-        pred = gold == results
+        pred = gold.strip() == results[0].strip()
        return {"em": pred}

    def aggregation(self):