Merge branch 'master' of https://github.com/EleutherAI/lm-evaluation-harness into pubmedqa

695d633d · jeffhsu3 · 0971cffe · 01630657 · 695d633d
Commit 695d633d authored Feb 06, 2021 by jeffhsu3
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 5 deletions

lm_eval/tasks/superglue.py lm_eval/tasks/superglue.py +4 -5

No files found.
--- a/lm_eval/tasks/superglue.py
+++ b/lm_eval/tasks/superglue.py
@@ -435,11 +435,10 @@ class SGWinogradSchemaChallenge(HFTask):
    def doc_to_text(self, doc):
        raw_passage = doc["text"]
-        passage = (
+        # NOTE: HuggingFace span indices are word-based not character-based.
-            raw_passage[:doc["span2_index"]]
+        pre = " ".join(raw_passage.split()[:doc["span2_index"]])
-            + "*{}*".format(doc["span2_text"])
+        post = raw_passage[len(pre) + len(doc["span2_text"]) + 1:]
-            + raw_passage[doc["span2_index"] + len(doc["span2_text"]):]
+        passage = pre + " *{}*".format(doc['span2_text']) + post
-        )
        noun = doc["span1_text"]
        pronoun = doc["span2_text"]
        text = (