Skip to content

GitLab

Unverified Commit 11f0e6d8 authored by Leo Gao's avatar Leo Gao Committed by GitHub
Browse files

Indicate that it's machine translated in task name and comments

parent 40bdb0c4
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 1 deletion
lm_eval/tasks/lambada_multilingual.py
View file @ 11f0e6d8
...@@ -4,6 +4,8 @@ from lm_eval.metrics import mean, perplexity ...@@ -4,6 +4,8 @@ from lm_eval.metrics import mean, perplexity
from lm_eval.utils import sh from lm_eval.utils import sh
from best_download import download_file from best_download import download_file
# This task is lambada but machine-translated to the other languages.
LANGS = ["en", "fr", "de", "it", "es"] LANGS = ["en", "fr", "de", "it", "es"]
CHECKSUMS = {"en": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226", CHECKSUMS = {"en": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226",
"fr": "941ec6a73dba7dc91c860bf493eb66a527cd430148827a4753a4535a046bf362", "fr": "941ec6a73dba7dc91c860bf493eb66a527cd430148827a4753a4535a046bf362",
...@@ -36,5 +38,5 @@ class MultilingualLAMBADA(lambada.LAMBADA): ...@@ -36,5 +38,5 @@ class MultilingualLAMBADA(lambada.LAMBADA):
def construct_tasks(): def construct_tasks():
tasks = {} tasks = {}
for lang in LANGS: for lang in LANGS:
tasks[f"lambada_{lang}"] = partial(MultilingualLAMBADA, lang=lang) tasks[f"lambada_mt_{lang}"] = partial(MultilingualLAMBADA, lang=lang)
return tasks return tasks
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment