Skip to content

GitLab

Commit 0fa316f8 authored by Leo Gao's avatar Leo Gao
Browse files

Make PerplexityTask fit the framework a bit better

parent b691c44f
Hide whitespace changes
Inline Side-by-side
Showing with 5 additions and 5 deletions
lm_eval/base.py
View file @ 0fa316f8
...@@ -315,10 +315,10 @@ class PerplexityTask(Task, abc.ABC): ...@@ -315,10 +315,10 @@ class PerplexityTask(Task, abc.ABC):
} }
def doc_to_text(self, doc): def doc_to_text(self, doc):
return doc return ""
def doc_to_target(self, doc): def doc_to_target(self, doc):
raise NotImplementedError() return doc
def construct_requests(self, doc, ctx): def construct_requests(self, doc, ctx):
assert not ctx assert not ctx
...@@ -327,12 +327,12 @@ class PerplexityTask(Task, abc.ABC): ...@@ -327,12 +327,12 @@ class PerplexityTask(Task, abc.ABC):
def process_results(self, doc, results): def process_results(self, doc, results):
loglikelihood, = results loglikelihood, = results
words = self.count_words(self.doc_to_text(doc)) words = self.count_words(self.doc_to_target(doc))
bytes = self.count_bytes(self.doc_to_text(doc)) bytes = self.count_bytes(self.doc_to_target(doc))
return { return {
"word_perplexity": (loglikelihood, words), "word_perplexity": (loglikelihood, words),
"byte_perplexity": (loglikelihood, bytes), "byte_perplexity": (loglikelihood, bytes),
"bits_per_byte": (-loglikelihood, self.count_bytes(self.doc_to_text(doc))) "bits_per_byte": (-loglikelihood, self.count_bytes(self.doc_to_target(doc)))
} }
def aggregation(self): def aggregation(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment