Skip to content

GitLab

Unverified Commit 0fe2e80a authored by Leo Gao's avatar Leo Gao Committed by GitHub
Browse files

Merge pull request #245 from bigscience-workshop/thomas/improve_boolq 

Nit on BoolQ
parents 5a53c368 f42a8d66
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 2 deletions
lm_eval/tasks/superglue.py
View file @ 0fe2e80a
...@@ -13,7 +13,7 @@ from ..utils import general_detokenize ...@@ -13,7 +13,7 @@ from ..utils import general_detokenize
class BoolQ(HFTask): class BoolQ(HFTask):
VERSION = 0 VERSION = 1
DATASET_PATH = "super_glue" DATASET_PATH = "super_glue"
DATASET_NAME = "boolq" DATASET_NAME = "boolq"
...@@ -31,7 +31,7 @@ class BoolQ(HFTask): ...@@ -31,7 +31,7 @@ class BoolQ(HFTask):
return "Read the following passages and answer each question with a yes or a no." return "Read the following passages and answer each question with a yes or a no."
def doc_to_text(self, doc): def doc_to_text(self, doc):
return f"{doc['passage']}\nQuestion: {doc['question']}\nAnswer:" return f"{doc['passage']}\nQuestion: {doc['question']}?\nAnswer:"
def doc_to_target(self, doc): def doc_to_target(self, doc):
return " " + yesno(doc['label']) return " " + yesno(doc['label'])
......
tests/testdata/boolq-v1-loglikelihood 0 → 100644
View file @ 0fe2e80a
6577e0d88572772ef08e64f624c0e3df0953286ae1f118ccef15623b59ffeabf
\ No newline at end of file
tests/testdata/boolq-v1-res.json 0 → 100644
View file @ 0fe2e80a
{"results": {"boolq": {"acc": 0.5048929663608562, "acc_stderr": 0.00874463623355505}}, "versions": {"boolq": 1}}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment