Merge pull request #41 from zphang/copa_update

Update COPA cause/effect

Merge pull request #41 from zphang/copa_update
Update COPA cause/effect
2338188f · Stella Biderman · GitHub · 3d5d794a · d5893cdc · 2338188f
Unverified Commit 2338188f authored Oct 22, 2020 by Stella Biderman Committed by GitHub Oct 22, 2020
Hide whitespace changes
Inline Side-by-side

Showing with 5 additions and 1 deletion

lm_eval/tasks/superglue.py lm_eval/tasks/superglue.py +5 -1

No files found.
--- a/lm_eval/tasks/superglue.py
+++ b/lm_eval/tasks/superglue.py
@@ -94,7 +94,11 @@ class Copa(HFTask):

    def doc_to_text(self, doc, include_target=True):
        # Drop the period
-        text = doc["premise"].strip()[:-1] + " because "
+        connector = {
+            "cause": "because",
+            "effect": "therefore",
+        }[doc["question"]]
+        text = doc["premise"].strip()[:-1] + f" {connector} "
        if include_target:
            correct_choice = doc["choice1"] if doc["label"] == 0 else doc["choice2"]
            # Connect the sentences