Skip to content

GitLab

Commit 2ced79f5 authored by Leo Gao's avatar Leo Gao
Browse files

Add "choices" to logiqa, hendrycksTest

parent 49ee4db2
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 2 deletions
lm_eval/tasks/hendrycks_test.py
View file @ 2ced79f5
...@@ -63,13 +63,14 @@ class GeneralHendrycksTest(MultipleChoiceTask): ...@@ -63,13 +63,14 @@ class GeneralHendrycksTest(MultipleChoiceTask):
def format_example(doc, choices): def format_example(doc, choices):
""" """
Question: <prompt> Question: <prompt>
Choices:
A. <choice1> A. <choice1>
B. <choice2> B. <choice2>
C. <choice3> C. <choice3>
D. <choice4> D. <choice4>
Answer: Answer:
""" """
prompt = "Question: " + doc[0] + "\n" prompt = "Question: " + doc[0] + "\nChoices:\n"
prompt += "".join([f"{choices[j]}. {doc[j+1]}\n" for j in range(4)]) prompt += "".join([f"{choices[j]}. {doc[j+1]}\n" for j in range(4)])
prompt += "Answer:" prompt += "Answer:"
return prompt return prompt
......
lm_eval/tasks/logiqa.py
View file @ 2ced79f5
...@@ -34,6 +34,7 @@ class LogiQA(MultipleChoiceTask): ...@@ -34,6 +34,7 @@ class LogiQA(MultipleChoiceTask):
""" """
Passage: <passage> Passage: <passage>
Question: <question> Question: <question>
Choices:
A. <choice1> A. <choice1>
B. <choice2> B. <choice2>
C. <choice3> C. <choice3>
...@@ -41,7 +42,7 @@ class LogiQA(MultipleChoiceTask): ...@@ -41,7 +42,7 @@ class LogiQA(MultipleChoiceTask):
Answer: Answer:
""" """
prompt = "Passage: " + doc["passage"] + "\n" prompt = "Passage: " + doc["passage"] + "\n"
prompt += "Question: " + doc["question"] + "\n" prompt += "Question: " + doc["question"] + "\nChoices:\n"
for choice, option in zip(choices, doc["options"]): for choice, option in zip(choices, doc["options"]):
prompt += f"{choice.upper()}. {option}\n" prompt += f"{choice.upper()}. {option}\n"
prompt += "Answer:" prompt += "Answer:"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment