update

lm_eval/tasks/xnli/utils.py

@@ -124,8 +124,12 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
                         "include": "xnli_common_yaml",
                         "dataset_name": lang,
                         "task": f"xnli_{lang}",
-                        "doc_to_text": f"{{{{premise}}}}, {QUESTION_WORD}?",
-                        "doc_to_choice": f"{{[\"{ENTAILMENT_LABEL}, {{{{hypothesis}}}}\", \"{NEUTRAL_LABEL}, {{{{hypothesis}}}}\", \"{CONTRADICTION_LABEL}, {{{{hypothesis}}}}\"]}}"
+                        "doc_to_text": "",
+                        "doc_to_choice": f"{{{{["
+                            f"""premise+\", {QUESTION_WORD}? {ENTAILMENT_LABEL}, \"+hypothesis,"""
+                            f"""premise+\", {QUESTION_WORD}? {NEUTRAL_LABEL}, \"+hypothesis,"""
+                            f"""premise+\", {QUESTION_WORD}? {CONTRADICTION_LABEL}, \"+hypothesis"""
+                            f"]}}}}"
                     },
                     f,
                     allow_unicode=True

lm_eval/tasks/xnli/xnli_ar.yaml

 # Generated by utils.py
 dataset_name: ar
-doc_to_choice: '{["نعم, {{hypothesis}}", "لذا, {{hypothesis}}", "رقم, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, صحيح?'
+doc_to_choice: '{{[premise+", صحيح? نعم, "+hypothesis,premise+", صحيح? لذا, "+hypothesis,premise+",
+  صحيح? رقم, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_ar

lm_eval/tasks/xnli/xnli_bg.yaml

 # Generated by utils.py
 dataset_name: bg
-doc_to_choice: '{["да, {{hypothesis}}", "така, {{hypothesis}}", "не, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, правилно?'
+doc_to_choice: '{{[premise+", правилно? да, "+hypothesis,premise+", правилно? така,
+  "+hypothesis,premise+", правилно? не, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_bg

lm_eval/tasks/xnli/xnli_common_yaml

@@ -8,7 +8,6 @@ dataset_name: null
 output_type: multiple_choice
 training_split: train
 validation_split: validation
-test_split: test
 doc_to_text: null
 doc_to_target: label
 doc_to_choice: null

lm_eval/tasks/xnli/xnli_de.yaml

 # Generated by utils.py
 dataset_name: de
-doc_to_choice: '{["Ja, {{hypothesis}}", "Auch, {{hypothesis}}", "Nein, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, richtig?'
+doc_to_choice: '{{[premise+", richtig? Ja, "+hypothesis,premise+", richtig? Auch,
+  "+hypothesis,premise+", richtig? Nein, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_de

lm_eval/tasks/xnli/xnli_el.yaml

 # Generated by utils.py
 dataset_name: el
-doc_to_choice: '{["Ναί, {{hypothesis}}", "Έτσι, {{hypothesis}}", "όχι, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, σωστός?'
+doc_to_choice: '{{[premise+", σωστός? Ναί, "+hypothesis,premise+", σωστός? Έτσι, "+hypothesis,premise+",
+  σωστός? όχι, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_el

lm_eval/tasks/xnli/xnli_en.yaml

 # Generated by utils.py
 dataset_name: en
-doc_to_choice: '{["Yes, {{hypothesis}}", "Also, {{hypothesis}}", "No, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, right?'
+doc_to_choice: '{{[premise+", right? Yes, "+hypothesis,premise+", right? Also, "+hypothesis,premise+",
+  right? No, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_en

lm_eval/tasks/xnli/xnli_es.yaml

 # Generated by utils.py
 dataset_name: es
-doc_to_choice: '{["Sí, {{hypothesis}}", "Asi que, {{hypothesis}}", "No, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, correcto?'
+doc_to_choice: '{{[premise+", correcto? Sí, "+hypothesis,premise+", correcto? Asi
+  que, "+hypothesis,premise+", correcto? No, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_es

lm_eval/tasks/xnli/xnli_fr.yaml

 # Generated by utils.py
 dataset_name: fr
-doc_to_choice: '{["Oui, {{hypothesis}}", "Aussi, {{hypothesis}}", "Non, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, correct?'
+doc_to_choice: '{{[premise+", correct? Oui, "+hypothesis,premise+", correct? Aussi,
+  "+hypothesis,premise+", correct? Non, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_fr

lm_eval/tasks/xnli/xnli_hi.yaml

 # Generated by utils.py
 dataset_name: hi
-doc_to_choice: '{["हाँ, {{hypothesis}}", "इसलिए, {{hypothesis}}", "नहीं, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, सही?'
+doc_to_choice: '{{[premise+", सही? हाँ, "+hypothesis,premise+", सही? इसलिए, "+hypothesis,premise+",
+  सही? नहीं, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_hi

lm_eval/tasks/xnli/xnli_ru.yaml

 # Generated by utils.py
 dataset_name: ru
-doc_to_choice: '{["Да, {{hypothesis}}", "Так, {{hypothesis}}", "Нет, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, правильно?'
+doc_to_choice: '{{[premise+", правильно? Да, "+hypothesis,premise+", правильно? Так,
+  "+hypothesis,premise+", правильно? Нет, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_ru

lm_eval/tasks/xnli/xnli_sw.yaml

 # Generated by utils.py
 dataset_name: sw
-doc_to_choice: '{["Ndiyo, {{hypothesis}}", "Hivyo, {{hypothesis}}", "Hapana, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, sahihi?'
+doc_to_choice: '{{[premise+", sahihi? Ndiyo, "+hypothesis,premise+", sahihi? Hivyo,
+  "+hypothesis,premise+", sahihi? Hapana, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_sw

lm_eval/tasks/xnli/xnli_th.yaml

 # Generated by utils.py
 dataset_name: th
-doc_to_choice: '{["ใช่, {{hypothesis}}", "ดังนั้น, {{hypothesis}}", "ไม่, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, ถูกต้อง?'
+doc_to_choice: '{{[premise+", ถูกต้อง? ใช่, "+hypothesis,premise+", ถูกต้อง? ดังนั้น,
+  "+hypothesis,premise+", ถูกต้อง? ไม่, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_th

lm_eval/tasks/xnli/xnli_tr.yaml

 # Generated by utils.py
 dataset_name: tr
-doc_to_choice: '{["Evet, {{hypothesis}}", "Böylece, {{hypothesis}}", "Hayır, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, doğru?'
+doc_to_choice: '{{[premise+", doğru? Evet, "+hypothesis,premise+", doğru? Böylece,
+  "+hypothesis,premise+", doğru? Hayır, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_tr

lm_eval/tasks/xnli/xnli_ur.yaml

 # Generated by utils.py
 dataset_name: ur
-doc_to_choice: '{["جی ہاں, {{hypothesis}}", "اس لئے, {{hypothesis}}", "نہیں, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, صحیح?'
+doc_to_choice: '{{[premise+", صحیح? جی ہاں, "+hypothesis,premise+", صحیح? اس لئے,
+  "+hypothesis,premise+", صحیح? نہیں, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_ur

lm_eval/tasks/xnli/xnli_vi.yaml

 # Generated by utils.py
 dataset_name: vi
-doc_to_choice: '{["Vâng, {{hypothesis}}", "Vì vậy, {{hypothesis}}", "Không, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, đúng?'
+doc_to_choice: '{{[premise+", đúng? Vâng, "+hypothesis,premise+", đúng? Vì vậy, "+hypothesis,premise+",
+  đúng? Không, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_vi

lm_eval/tasks/xnli/xnli_zh.yaml

 # Generated by utils.py
 dataset_name: zh
-doc_to_choice: '{["是的, {{hypothesis}}", "所以, {{hypothesis}}", "不是的, {{hypothesis}}"]}'
-doc_to_text: '{{premise}}, 正确?'
+doc_to_choice: '{{[premise+", 正确? 是的, "+hypothesis,premise+", 正确? 所以, "+hypothesis,premise+",
+  正确? 不是的, "+hypothesis]}}'
+doc_to_text: ''
 include: xnli_common_yaml
 task: xnli_zh