Skip to content

GitLab

Commit 1b69fb43 authored by jon-tow's avatar jon-tow
Browse files

fix: use formatting from paper

parent 48baeea9
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
lm_eval/tasks/siqa.py
View file @ 1b69fb43
...@@ -57,10 +57,10 @@ class SIQA(MultipleChoiceTask): ...@@ -57,10 +57,10 @@ class SIQA(MultipleChoiceTask):
def _process_doc(self, doc): def _process_doc(self, doc):
return { return {
"query": f"{doc['context']}\nQuestion: {doc['question']}", "query": f"{doc['context']} {doc['question']}",
"choices": [doc['answerA'], doc['answerB'], doc['answerC']], "choices": [doc['answerA'], doc['answerB'], doc['answerC']],
"gold": int(doc['label']) - 1, # `-1` because the labels are 1-indexed. "gold": int(doc['label']) - 1, # `-1` because the labels are 1-indexed.
} }
def doc_to_text(self, doc): def doc_to_text(self, doc):
return doc["query"] + "\nAnswer:" return doc["query"]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment