Skip to content

GitLab

Commit 7ab64701 authored by Dirk Groeneveld's avatar Dirk Groeneveld
Browse files

Use `_process_doc` like the other tasks do

parent a080b5ad
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 4 deletions
lm_eval/tasks/wikitext.py
View file @ 7ab64701
...@@ -76,15 +76,15 @@ class WikiText(PerplexityTask): ...@@ -76,15 +76,15 @@ class WikiText(PerplexityTask):
return True return True
def training_docs(self): def training_docs(self):
return map(self._load_doc, self.dataset["train"]) return map(self._process_doc, self.dataset["train"])
def validation_docs(self): def validation_docs(self):
return map(self._load_doc, self.dataset["validation"]) return map(self._process_doc, self.dataset["validation"])
def test_docs(self): def test_docs(self):
return map(self._load_doc, self.dataset["test"]) return map(self._process_doc, self.dataset["test"])
def _load_doc(self, doc): def _process_doc(self, doc):
return doc["page"] return doc["page"]
def doc_to_target(self, doc): def doc_to_target(self, doc):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment