Skip to content

GitLab

Unverified Commit 56a4e794 authored by Lintang Sutawika's avatar Lintang Sutawika Committed by GitHub
Browse files

formatting (#2104)

parent 9884ad6e
Show whitespace changes
Inline Side-by-side
Showing with 103 additions and 93 deletions
lm_eval/tasks/afrimmlu/translate/utils.py
View file @ 56a4e794
...@@ -21,12 +21,14 @@ def doc_to_text(doc): ...@@ -21,12 +21,14 @@ def doc_to_text(doc):
Answer: """ Answer: """
choices = eval(doc["choices"]) choices = eval(doc["choices"])
text = output.format(subject=doc['subject'], text = output.format(
question=doc['question'], subject=doc["subject"],
question=doc["question"],
choice1=choices[0], choice1=choices[0],
choice2=choices[1], choice2=choices[1],
choice3=choices[2], choice3=choices[2],
choice4=choices[3]) choice4=choices[3],
)
return text return text
......
lm_eval/tasks/afrimmlu/utils.py
View file @ 56a4e794
...@@ -21,12 +21,14 @@ def doc_to_text(doc): ...@@ -21,12 +21,14 @@ def doc_to_text(doc):
Answer: """ Answer: """
choices = eval(doc["choices"]) choices = eval(doc["choices"])
text = output.format(subject=doc['subject'], text = output.format(
question=doc['question'], subject=doc["subject"],
question=doc["question"],
choice1=choices[0], choice1=choices[0],
choice2=choices[1], choice2=choices[1],
choice3=choices[2], choice3=choices[2],
choice4=choices[3]) choice4=choices[3],
)
return text return text
......
lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py
View file @ 56a4e794
...@@ -2,11 +2,7 @@ from sklearn.metrics import f1_score ...@@ -2,11 +2,7 @@ from sklearn.metrics import f1_score
def doc_to_target(doc): def doc_to_target(doc):
replacements = { replacements = {0: "True", 1: "Neither", 2: "False"}
0: 'True',
1: 'Neither',
2: 'False'
}
return replacements[doc["label"]] return replacements[doc["label"]]
......
lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml
View file @ 56a4e794
...@@ -2,4 +2,3 @@ ...@@ -2,4 +2,3 @@
dataset_name: amh dataset_name: amh
include: afrixnli_translate_yaml include: afrixnli_translate_yaml
task: afrixnli_translate_amh task: afrixnli_translate_amh
lm_eval/tasks/afrixnli/anli prompt/translate/utils.py
View file @ 56a4e794
...@@ -2,11 +2,7 @@ from sklearn.metrics import f1_score ...@@ -2,11 +2,7 @@ from sklearn.metrics import f1_score
def doc_to_target(doc): def doc_to_target(doc):
replacements = { replacements = {0: "True", 1: "Neither", 2: "False"}
0: 'True',
1: 'Neither',
2: 'False'
}
return replacements[doc["label"]] return replacements[doc["label"]]
......
lm_eval/tasks/afrixnli/lai prompt/direct/utils.py
View file @ 56a4e794
...@@ -10,17 +10,12 @@ def doc_to_text(doc): ...@@ -10,17 +10,12 @@ def doc_to_text(doc):
Is it entailment, contradiction, or neutral?""" Is it entailment, contradiction, or neutral?"""
text = output.format(premise=doc['premise'], text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"])
hypothesis=doc['hypothesis'])
return text return text
def doc_to_target(doc): def doc_to_target(doc):
replacements = { replacements = {0: "entailment", 1: "neutral", 2: "contradiction"}
0: 'entailment',
1: 'neutral',
2: 'contradiction'
}
return replacements[doc["label"]] return replacements[doc["label"]]
......
lm_eval/tasks/afrixnli/lai prompt/translate/utils.py
View file @ 56a4e794
...@@ -10,17 +10,12 @@ def doc_to_text(doc): ...@@ -10,17 +10,12 @@ def doc_to_text(doc):
Is it entailment, contradiction, or neutral?""" Is it entailment, contradiction, or neutral?"""
text = output.format(premise=doc['premise'], text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"])
hypothesis=doc['hypothesis'])
return text return text
def doc_to_target(doc): def doc_to_target(doc):
replacements = { replacements = {0: "entailment", 1: "neutral", 2: "contradiction"}
0: 'entailment',
1: 'neutral',
2: 'contradiction'
}
return replacements[doc["label"]] return replacements[doc["label"]]
......
lm_eval/tasks/afrixnli/utils.py
View file @ 56a4e794
import yaml
import argparse import argparse
import yaml
class FunctionTag: class FunctionTag:
def __init__(self, value): def __init__(self, value):
...@@ -12,110 +13,110 @@ LANGUAGES = { ...@@ -12,110 +13,110 @@ LANGUAGES = {
"QUESTION_WORD": "ትክክል", "QUESTION_WORD": "ትክክል",
"ENTAILMENT_LABEL": "አዎ", "ENTAILMENT_LABEL": "አዎ",
"NEUTRAL_LABEL": "እንዲሁም", "NEUTRAL_LABEL": "እንዲሁም",
"CONTRADICTION_LABEL": "አይ" "CONTRADICTION_LABEL": "አይ",
}, },
"eng": { "eng": {
"QUESTION_WORD": "Right", "QUESTION_WORD": "Right",
"ENTAILMENT_LABEL": "Yes", "ENTAILMENT_LABEL": "Yes",
"NEUTRAL_LABEL": "Also", "NEUTRAL_LABEL": "Also",
"CONTRADICTION_LABEL": "No" "CONTRADICTION_LABEL": "No",
}, },
"ewe": { "ewe": {
"QUESTION_WORD": "Esɔ gbe", "QUESTION_WORD": "Esɔ gbe",
"ENTAILMENT_LABEL": "Ɛ̃", "ENTAILMENT_LABEL": "Ɛ̃",
"NEUTRAL_LABEL": "Hã", "NEUTRAL_LABEL": "Hã",
"CONTRADICTION_LABEL": "Ao" "CONTRADICTION_LABEL": "Ao",
}, },
"fra": { "fra": {
"QUESTION_WORD": "correct", "QUESTION_WORD": "correct",
"ENTAILMENT_LABEL": "Oui", "ENTAILMENT_LABEL": "Oui",
"NEUTRAL_LABEL": "Aussi", "NEUTRAL_LABEL": "Aussi",
"CONTRADICTION_LABEL": "Non" "CONTRADICTION_LABEL": "Non",
}, },
"hau": { "hau": {
"QUESTION_WORD": "Daidai", "QUESTION_WORD": "Daidai",
"ENTAILMENT_LABEL": "Ee", "ENTAILMENT_LABEL": "Ee",
"NEUTRAL_LABEL": "Haka kuma", "NEUTRAL_LABEL": "Haka kuma",
"CONTRADICTION_LABEL": "A'a" "CONTRADICTION_LABEL": "A'a",
}, },
"ibo": { "ibo": {
"QUESTION_WORD": "Ziri ezi", "QUESTION_WORD": "Ziri ezi",
"ENTAILMENT_LABEL": "Éè", "ENTAILMENT_LABEL": "Éè",
"NEUTRAL_LABEL": "Ọzọkwa", "NEUTRAL_LABEL": "Ọzọkwa",
"CONTRADICTION_LABEL": "Mba" "CONTRADICTION_LABEL": "Mba",
}, },
"kin": { "kin": {
"QUESTION_WORD": "Nibyo", "QUESTION_WORD": "Nibyo",
"ENTAILMENT_LABEL": "Yego", "ENTAILMENT_LABEL": "Yego",
"NEUTRAL_LABEL": "Na none", "NEUTRAL_LABEL": "Na none",
"CONTRADICTION_LABEL": "Oya" "CONTRADICTION_LABEL": "Oya",
}, },
"lin": { "lin": {
"QUESTION_WORD": "Malamu", "QUESTION_WORD": "Malamu",
"ENTAILMENT_LABEL": "Iyo", "ENTAILMENT_LABEL": "Iyo",
"NEUTRAL_LABEL": "Lisusu", "NEUTRAL_LABEL": "Lisusu",
"CONTRADICTION_LABEL": "Te" "CONTRADICTION_LABEL": "Te",
}, },
"lug": { "lug": {
"QUESTION_WORD": "Kituufu", "QUESTION_WORD": "Kituufu",
"ENTAILMENT_LABEL": "Yee", "ENTAILMENT_LABEL": "Yee",
"NEUTRAL_LABEL": "N’ekirala", "NEUTRAL_LABEL": "N’ekirala",
"CONTRADICTION_LABEL": "Nedda" "CONTRADICTION_LABEL": "Nedda",
}, },
"orm": { "orm": {
"QUESTION_WORD": "Sirrii", "QUESTION_WORD": "Sirrii",
"ENTAILMENT_LABEL": "Eeyyee", "ENTAILMENT_LABEL": "Eeyyee",
"NEUTRAL_LABEL": "Akkasumas", "NEUTRAL_LABEL": "Akkasumas",
"CONTRADICTION_LABEL": "Lakki" "CONTRADICTION_LABEL": "Lakki",
}, },
"sna": { "sna": {
"QUESTION_WORD": "Chokwadi", "QUESTION_WORD": "Chokwadi",
"ENTAILMENT_LABEL": "Hongu", "ENTAILMENT_LABEL": "Hongu",
"NEUTRAL_LABEL": "Uye", "NEUTRAL_LABEL": "Uye",
"CONTRADICTION_LABEL": "Kwete" "CONTRADICTION_LABEL": "Kwete",
}, },
"sot": { "sot": {
"QUESTION_WORD": "Nepile", "QUESTION_WORD": "Nepile",
"ENTAILMENT_LABEL": "E", "ENTAILMENT_LABEL": "E",
"NEUTRAL_LABEL": "Hape", "NEUTRAL_LABEL": "Hape",
"CONTRADICTION_LABEL": "Tjhe" "CONTRADICTION_LABEL": "Tjhe",
}, },
"swa": { "swa": {
"QUESTION_WORD": "Sahihi", "QUESTION_WORD": "Sahihi",
"ENTAILMENT_LABEL": "Ndiyo", "ENTAILMENT_LABEL": "Ndiyo",
"NEUTRAL_LABEL": "Pia", "NEUTRAL_LABEL": "Pia",
"CONTRADICTION_LABEL": "Hapana" "CONTRADICTION_LABEL": "Hapana",
}, },
"twi": { "twi": {
"QUESTION_WORD": "Nifa", "QUESTION_WORD": "Nifa",
"ENTAILMENT_LABEL": "Aane", "ENTAILMENT_LABEL": "Aane",
"NEUTRAL_LABEL": "Anaasɛ", "NEUTRAL_LABEL": "Anaasɛ",
"CONTRADICTION_LABEL": "Daabi" "CONTRADICTION_LABEL": "Daabi",
}, },
"wol": { "wol": {
"QUESTION_WORD": "Dëgg", "QUESTION_WORD": "Dëgg",
"ENTAILMENT_LABEL": "Waaw", "ENTAILMENT_LABEL": "Waaw",
"NEUTRAL_LABEL": "Itam", "NEUTRAL_LABEL": "Itam",
"CONTRADICTION_LABEL": "Déet" "CONTRADICTION_LABEL": "Déet",
}, },
"xho": { "xho": {
"QUESTION_WORD": "Ichanekile", "QUESTION_WORD": "Ichanekile",
"ENTAILMENT_LABEL": "Ewe", "ENTAILMENT_LABEL": "Ewe",
"NEUTRAL_LABEL": "Kananjalo", "NEUTRAL_LABEL": "Kananjalo",
"CONTRADICTION_LABEL": "Hayi" "CONTRADICTION_LABEL": "Hayi",
}, },
"yor": { "yor": {
"QUESTION_WORD": "Òótọ́", "QUESTION_WORD": "Òótọ́",
"ENTAILMENT_LABEL": "Bẹ́ẹ̀ni", "ENTAILMENT_LABEL": "Bẹ́ẹ̀ni",
"NEUTRAL_LABEL": "Àti pé", "NEUTRAL_LABEL": "Àti pé",
"CONTRADICTION_LABEL": "Rárá" "CONTRADICTION_LABEL": "Rárá",
}, },
"zul": { "zul": {
"QUESTION_WORD": "Kulungile", "QUESTION_WORD": "Kulungile",
"ENTAILMENT_LABEL": "Yebo", "ENTAILMENT_LABEL": "Yebo",
"NEUTRAL_LABEL": "Futhi", "NEUTRAL_LABEL": "Futhi",
"CONTRADICTION_LABEL": "Cha" "CONTRADICTION_LABEL": "Cha",
} },
} }
...@@ -127,8 +128,26 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: ...@@ -127,8 +128,26 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
:param overwrite: Whether to overwrite files if they already exist. :param overwrite: Whether to overwrite files if they already exist.
""" """
err = [] err = []
languages = ['eng', 'amh', 'ibo', 'fra', 'sna', 'wol', 'ewe', 'lin', 'lug', 'xho', 'kin', 'twi', 'zul', 'orm', languages = [
'yor', 'hau', 'sot', 'swa'] "eng",
"amh",
"ibo",
"fra",
"sna",
"wol",
"ewe",
"lin",
"lug",
"xho",
"kin",
"twi",
"zul",
"orm",
"yor",
"hau",
"sot",
"swa",
]
for lang in languages: for lang in languages:
try: try:
if mode == "native-direct": if mode == "native-direct":
...@@ -141,7 +160,9 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: ...@@ -141,7 +160,9 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
task_name = f"afrixnli_native_direct_{lang}" task_name = f"afrixnli_native_direct_{lang}"
yaml_template = "afrixnli_native_direct_yaml" yaml_template = "afrixnli_native_direct_yaml"
with open( with open(
f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8" f"{output_dir}/{file_name}",
"w" if overwrite else "x",
encoding="utf8",
) as f: ) as f:
f.write("# Generated by utils.py\n") f.write("# Generated by utils.py\n")
yaml.dump( yaml.dump(
...@@ -163,14 +184,16 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: ...@@ -163,14 +184,16 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
task_name = f"afrixnli_{mode}_{lang}" task_name = f"afrixnli_{mode}_{lang}"
yaml_template = f"afrixnli_{mode}_yaml" yaml_template = f"afrixnli_{mode}_yaml"
with open( with open(
f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8" f"{output_dir}/{file_name}",
"w" if overwrite else "x",
encoding="utf8",
) as f: ) as f:
f.write("# Generated by utils.py\n") f.write("# Generated by utils.py\n")
yaml.dump( yaml.dump(
{ {
"include": yaml_template, "include": yaml_template,
"task": task_name, "task": task_name,
"dataset_name": lang "dataset_name": lang,
}, },
f, f,
allow_unicode=True, allow_unicode=True,
...@@ -195,7 +218,9 @@ def main() -> None: ...@@ -195,7 +218,9 @@ def main() -> None:
help="Overwrite files if they already exist", help="Overwrite files if they already exist",
) )
parser.add_argument( parser.add_argument(
"--output-dir", default="./manual/translate", help="Directory to write yaml files to" "--output-dir",
default="./manual/translate",
help="Directory to write yaml files to",
) )
parser.add_argument( parser.add_argument(
"--mode", "--mode",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment