Skip to content

GitLab

Commit 5d431ba9 authored by Jon Tow's avatar Jon Tow
Browse files

Update `doc_to_text` to show candidate fill-ins

parent 65c46d22
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 1 deletion
lm_eval/tasks/hellaswag.py
View file @ 5d431ba9
import re
import numpy as np import numpy as np
from ..base import rf, mean from ..base import rf, mean
from . common import HFTask from . common import HFTask
...@@ -34,7 +35,9 @@ class HellaSwag(HFTask): ...@@ -34,7 +35,9 @@ class HellaSwag(HFTask):
"plausibly completes the situation." "plausibly completes the situation."
def doc_to_text(self, doc): def doc_to_text(self, doc):
return doc['activity_label'] + ': ' + doc['ctx'] + '\n' text = doc['activity_label'] + ': ' + doc['ctx'] + '\n'
text += ''.join([ending + "\n" for ending in doc['endings']])
return text + "\n"
def doc_to_target(self, doc): def doc_to_target(self, doc):
letter_answer = doc['label'] letter_answer = doc['label']
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment