xnli reformat

lm_eval/tasks/xnli/utils.py

@@ -9,95 +9,95 @@ import yaml
 # A yaml file is generated by this script for each language.
 
 LANGUAGES = {
-    "ar": { # Arabic
+    "ar": {  # Arabic
         "QUESTION_WORD": "صحيح",
         "ENTAILMENT_LABEL": "نعم",
         "NEUTRAL_LABEL": "لذا",
-        "CONTRADICTION_LABEL": "رقم"
+        "CONTRADICTION_LABEL": "رقم",
     },
-    "bg": { # Bulgarian
+    "bg": {  # Bulgarian
         "QUESTION_WORD": "правилно",
         "ENTAILMENT_LABEL": "да",
         "NEUTRAL_LABEL": "така",
-        "CONTRADICTION_LABEL": "не"
+        "CONTRADICTION_LABEL": "не",
     },
-    "de": { # German
+    "de": {  # German
         "QUESTION_WORD": "richtig",
         "ENTAILMENT_LABEL": "Ja",
         "NEUTRAL_LABEL": "Auch",
-        "CONTRADICTION_LABEL": "Nein"
+        "CONTRADICTION_LABEL": "Nein",
     },
-    "el": { # Greek
+    "el": {  # Greek
         "QUESTION_WORD": "σωστός",
         "ENTAILMENT_LABEL": "Ναί",
         "NEUTRAL_LABEL": "Έτσι",
-        "CONTRADICTION_LABEL": "όχι"
+        "CONTRADICTION_LABEL": "όχι",
     },
-    "en": { # English
+    "en": {  # English
         "QUESTION_WORD": "right",
         "ENTAILMENT_LABEL": "Yes",
         "NEUTRAL_LABEL": "Also",
-        "CONTRADICTION_LABEL": "No"
+        "CONTRADICTION_LABEL": "No",
     },
-    "es": { # Spanish
+    "es": {  # Spanish
         "QUESTION_WORD": "correcto",
         "ENTAILMENT_LABEL": "Sí",
         "NEUTRAL_LABEL": "Asi que",
-        "CONTRADICTION_LABEL": "No"
+        "CONTRADICTION_LABEL": "No",
     },
-    "fr": { # French
+    "fr": {  # French
         "QUESTION_WORD": "correct",
         "ENTAILMENT_LABEL": "Oui",
         "NEUTRAL_LABEL": "Aussi",
-        "CONTRADICTION_LABEL": "Non"
+        "CONTRADICTION_LABEL": "Non",
     },
-    "hi": { # Hindi
+    "hi": {  # Hindi
         "QUESTION_WORD": "सही",
         "ENTAILMENT_LABEL": "हाँ",
         "NEUTRAL_LABEL": "इसलिए",
-        "CONTRADICTION_LABEL": "नहीं"
+        "CONTRADICTION_LABEL": "नहीं",
     },
-    "ru": { # Russian
+    "ru": {  # Russian
         "QUESTION_WORD": "правильно",
         "ENTAILMENT_LABEL": "Да",
         "NEUTRAL_LABEL": "Так",
-        "CONTRADICTION_LABEL": "Нет"
+        "CONTRADICTION_LABEL": "Нет",
     },
-    "sw": { # Swahili
+    "sw": {  # Swahili
         "QUESTION_WORD": "sahihi",
         "ENTAILMENT_LABEL": "Ndiyo",
         "NEUTRAL_LABEL": "Hivyo",
-        "CONTRADICTION_LABEL": "Hapana"
+        "CONTRADICTION_LABEL": "Hapana",
     },
-    "th": { # Thai
+    "th": {  # Thai
         "QUESTION_WORD": "ถูกต้อง",
         "ENTAILMENT_LABEL": "ใช่",
         "NEUTRAL_LABEL": "ดังนั้น",
-        "CONTRADICTION_LABEL": "ไม่"
+        "CONTRADICTION_LABEL": "ไม่",
     },
-    "tr": { # Turkish
+    "tr": {  # Turkish
         "QUESTION_WORD": "doğru",
         "ENTAILMENT_LABEL": "Evet",
         "NEUTRAL_LABEL": "Böylece",
-        "CONTRADICTION_LABEL": "Hayır"
+        "CONTRADICTION_LABEL": "Hayır",
     },
-    "ur": { # Urdu
+    "ur": {  # Urdu
         "QUESTION_WORD": "صحیح",
         "ENTAILMENT_LABEL": "جی ہاں",
         "NEUTRAL_LABEL": "اس لئے",
-        "CONTRADICTION_LABEL": "نہیں"
+        "CONTRADICTION_LABEL": "نہیں",
     },
-    "vi": { # Vietnamese
+    "vi": {  # Vietnamese
         "QUESTION_WORD": "đúng",
         "ENTAILMENT_LABEL": "Vâng",
         "NEUTRAL_LABEL": "Vì vậy",
-        "CONTRADICTION_LABEL": "Không"
+        "CONTRADICTION_LABEL": "Không",
     },
-    "zh": { # Chinese
+    "zh": {  # Chinese
         "QUESTION_WORD": "正确",
         "ENTAILMENT_LABEL": "是的",
         "NEUTRAL_LABEL": "所以",
-        "CONTRADICTION_LABEL": "不是的"
+        "CONTRADICTION_LABEL": "不是的",
     },
 }
 
@@ -117,7 +117,9 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
             ENTAILMENT_LABEL = LANGUAGES[lang]["ENTAILMENT_LABEL"]
             NEUTRAL_LABEL = LANGUAGES[lang]["NEUTRAL_LABEL"]
             CONTRADICTION_LABEL = LANGUAGES[lang]["CONTRADICTION_LABEL"]
-            with open(f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding='utf8') as f:
+            with open(
+                f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8"
+            ) as f:
                 f.write("# Generated by utils.py\n")
                 yaml.dump(
                     {
@@ -126,13 +128,13 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
                         "task": f"xnli_{lang}",
                         "doc_to_text": "",
                         "doc_to_choice": f"{{{{["
-                            f"""premise+\", {QUESTION_WORD}? {ENTAILMENT_LABEL}, \"+hypothesis,"""
-                            f"""premise+\", {QUESTION_WORD}? {NEUTRAL_LABEL}, \"+hypothesis,"""
-                            f"""premise+\", {QUESTION_WORD}? {CONTRADICTION_LABEL}, \"+hypothesis"""
-                            f"]}}}}"
+                        f"""premise+\", {QUESTION_WORD}? {ENTAILMENT_LABEL}, \"+hypothesis,"""
+                        f"""premise+\", {QUESTION_WORD}? {NEUTRAL_LABEL}, \"+hypothesis,"""
+                        f"""premise+\", {QUESTION_WORD}? {CONTRADICTION_LABEL}, \"+hypothesis"""
+                        f"]}}}}",
                     },
                     f,
-                    allow_unicode=True
+                    allow_unicode=True,
                 )
         except FileExistsError:
             err.append(file_name)