Skip to content

GitLab

Commit 2f5f42c6 authored by Leo Gao's avatar Leo Gao
Browse files

Add MultipleChoiceTest target

parent 7422d593
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 0 deletions
lm_eval/base.py
View file @ 2f5f42c6
...@@ -181,6 +181,9 @@ class Task(abc.ABC): ...@@ -181,6 +181,9 @@ class Task(abc.ABC):
class MultipleChoiceTask(Task): class MultipleChoiceTask(Task):
def doc_to_target(self, doc):
return " " + doc['choices'][doc['gold']]
def construct_requests(self, doc, ctx): def construct_requests(self, doc, ctx):
lls = [ lls = [
rf.loglikelihood(ctx, " {}".format(choice))[0] rf.loglikelihood(ctx, " {}".format(choice))[0]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment