Skip to content

GitLab

Commit 00038fea authored by Leo Gao's avatar Leo Gao
Browse files

Q/A instead of Question/Answer

parent 6c816cb7
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
lm_eval/tasks/webqs.py
View file @ 00038fea
...@@ -19,12 +19,12 @@ class WebQs(HFNLPTask): ...@@ -19,12 +19,12 @@ class WebQs(HFNLPTask):
def doc_to_text(self, doc, include_target=True): def doc_to_text(self, doc, include_target=True):
print(doc) print(doc)
q = "Question: " + doc['question'] + '\n' q = "Q: " + doc['question'] + '\n'
# this picks one answer to be the "correct" one, despite sometimes # this picks one answer to be the "correct" one, despite sometimes
# multiple correct answers being possible. # multiple correct answers being possible.
# TODO: make sure we're actually handling multi-answer correctly # TODO: make sure we're actually handling multi-answer correctly
a = "Answer:" + ((" " + doc['answers'][0]) if include_target else '') a = "A:" + ((" " + doc['answers'][0]) if include_target else '')
return q + a return q + a
def evaluate(self, docs, lm, provide_description, num_fewshot): def evaluate(self, docs, lm, provide_description, num_fewshot):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment