Irokobench: Benchmark Dataset for African languages (#2042)

* add afrixnli to task

* add chat completion

* remove chat completion -untested

* afrimmlu added

* afrimmlu folder update

* afrimmlu folder update

* updated prompt

* remove print

* add afrimgsm -direct

* add squad metric

* fix bash script

* remove direct util, update common yaml

* remove print

* add few show. metric fixes

* fix direct path, add bash script for gpt models

* added transate test

* update afrixnli tasks

* update afrixnli tasks

* update metrics for afrixnli

* prompt translations fix

* prompt translations fix

* filter and metric fix -mgsm

* remove squad metric

* remove squad metric

* add f1 score to mgsm

* add f1 score to mgsm

* update native-direct with lin

* change f1 function

* add lin to utils

* add utils

* remove test limit

* remove test configs

* add swahili to mmlu

* change eng to ewe in ewe yaml mmlu

* add squad metric to mgsm, remove whitespace filter

* added translate test

* added afrixnli_translate

* fix exact match valueError

* fix exact match valueError

* restructure mmlu folder

* spacing

* remove afrimmlu_translate folder

* add utility

* format task name, clean ups

* modefied mgsm

* update on afrimgsm

* update on afrimgsm

* removed utils

* other mgsm varieties

* other mgsm varieties

* adding trasnslate direct

* Update translate_direct_yaml

* add manual xnli prompt, add multichoice for openai models, and adapt multichoice metric for openai model

* edit for open models

* Update translate_direct_yaml

* add verbalizer for xnli

* change xnli from multiple choice to generate

* add manual accuracy scores

* revert xnli to multiple choice

* change afrimgsm utils

* revert xnli to multiple_choice

* cleanups and readmes

* remove openai fixes and unused regex

* pr review changes

* revert metrics.py, task.py and extraction.py to main version

---------
Co-authored-by: Israel Abebe Azime <azime@cg.uni-saarland.de>
Co-authored-by: Israel Abebe Azime <se.israel.abebe@gmail.com>

lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml

+# Generated by utils.py
+dataset_name: wol
+include: afrixnli_manual_translate_yaml
+task: afrixnli_manual_translate_wol

lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml

+# Generated by utils.py
+dataset_name: xho
+include: afrixnli_manual_translate_yaml
+task: afrixnli_manual_translate_xho

lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml

+group:
+    - xnli
+    - afrixnli
+    - afrixnli_manual_direct
+dataset_path: masakhane/afrixnli-translate-test
+dataset_name: null
+output_type: multiple_choice
+test_split: test
+doc_to_text: !function utils.doc_to_text
+doc_to_target: !function utils.doc_to_target
+doc_to_choice:
+  - "entailment"
+  - "neutral"
+  - "contradiction"
+should_decontaminate: true
+doc_to_decontamination_query: premise
+metric_list:
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
+    average: weighted
+    higher_is_better: True
+    ignore_case: true
+    ignore_punctuation: true
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 1.0

lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml

+# Generated by utils.py
+dataset_name: yor
+include: afrixnli_manual_translate_yaml
+task: afrixnli_manual_translate_yor

lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml

+# Generated by utils.py
+dataset_name: zul
+include: afrixnli_manual_translate_yaml
+task: afrixnli_manual_translate_zul

lm_eval/tasks/afrixnli/lai prompt/translate/utils.py

+from sklearn.metrics import f1_score
+
+
+def doc_to_text(doc):
+    output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise 
+    and hypothesis. The answer should be exact entailment, contradiction, or neutral.
+    
+    Premise: {premise}
+    Hypothesis: {hypothesis}
+    
+    Is it entailment, contradiction, or neutral?"""
+
+    text = output.format(premise=doc['premise'],
+                         hypothesis=doc['hypothesis'])
+    return text
+
+
+def doc_to_target(doc):
+    replacements = {
+        0: 'entailment',
+        1: 'neutral',
+        2: 'contradiction'
+    }
+    return replacements[doc["label"]]
+
+
+def weighted_f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = f1_score(golds, preds, average="weighted")
+    return fscore

lm_eval/tasks/afrixnli/utils.py

+import yaml
+import argparse
+
+
+class FunctionTag:
+    def __init__(self, value):
+        self.value = value
+
+
+LANGUAGES = {
+    "amh": {
+        "QUESTION_WORD": "ትክክል",
+        "ENTAILMENT_LABEL": "አዎ",
+        "NEUTRAL_LABEL": "እንዲሁም",
+        "CONTRADICTION_LABEL": "አይ"
+    },
+    "eng": {
+        "QUESTION_WORD": "Right",
+        "ENTAILMENT_LABEL": "Yes",
+        "NEUTRAL_LABEL": "Also",
+        "CONTRADICTION_LABEL": "No"
+    },
+    "ewe": {
+        "QUESTION_WORD": "Esɔ gbe",
+        "ENTAILMENT_LABEL": "Ɛ̃",
+        "NEUTRAL_LABEL": "Hã",
+        "CONTRADICTION_LABEL": "Ao"
+    },
+    "fra": {
+        "QUESTION_WORD": "correct",
+        "ENTAILMENT_LABEL": "Oui",
+        "NEUTRAL_LABEL": "Aussi",
+        "CONTRADICTION_LABEL": "Non"
+    },
+    "hau": {
+        "QUESTION_WORD": "Daidai",
+        "ENTAILMENT_LABEL": "Ee",
+        "NEUTRAL_LABEL": "Haka kuma",
+        "CONTRADICTION_LABEL": "A'a"
+    },
+    "ibo": {
+        "QUESTION_WORD": "Ziri ezi",
+        "ENTAILMENT_LABEL": "Éè",
+        "NEUTRAL_LABEL": "Ọzọkwa",
+        "CONTRADICTION_LABEL": "Mba"
+    },
+    "kin": {
+        "QUESTION_WORD": "Nibyo",
+        "ENTAILMENT_LABEL": "Yego",
+        "NEUTRAL_LABEL": "Na none",
+        "CONTRADICTION_LABEL": "Oya"
+    },
+    "lin": {
+        "QUESTION_WORD": "Malamu",
+        "ENTAILMENT_LABEL": "Iyo",
+        "NEUTRAL_LABEL": "Lisusu",
+        "CONTRADICTION_LABEL": "Te"
+    },
+    "lug": {
+        "QUESTION_WORD": "Kituufu",
+        "ENTAILMENT_LABEL": "Yee",
+        "NEUTRAL_LABEL": "N’ekirala",
+        "CONTRADICTION_LABEL": "Nedda"
+    },
+    "orm": {
+        "QUESTION_WORD": "Sirrii",
+        "ENTAILMENT_LABEL": "Eeyyee",
+        "NEUTRAL_LABEL": "Akkasumas",
+        "CONTRADICTION_LABEL": "Lakki"
+    },
+    "sna": {
+        "QUESTION_WORD": "Chokwadi",
+        "ENTAILMENT_LABEL": "Hongu",
+        "NEUTRAL_LABEL": "Uye",
+        "CONTRADICTION_LABEL": "Kwete"
+    },
+    "sot": {
+        "QUESTION_WORD": "Nepile",
+        "ENTAILMENT_LABEL": "E",
+        "NEUTRAL_LABEL": "Hape",
+        "CONTRADICTION_LABEL": "Tjhe"
+    },
+    "swa": {
+        "QUESTION_WORD": "Sahihi",
+        "ENTAILMENT_LABEL": "Ndiyo",
+        "NEUTRAL_LABEL": "Pia",
+        "CONTRADICTION_LABEL": "Hapana"
+    },
+    "twi": {
+        "QUESTION_WORD": "Nifa",
+        "ENTAILMENT_LABEL": "Aane",
+        "NEUTRAL_LABEL": "Anaasɛ",
+        "CONTRADICTION_LABEL": "Daabi"
+    },
+    "wol": {
+        "QUESTION_WORD": "Dëgg",
+        "ENTAILMENT_LABEL": "Waaw",
+        "NEUTRAL_LABEL": "Itam",
+        "CONTRADICTION_LABEL": "Déet"
+    },
+    "xho": {
+        "QUESTION_WORD": "Ichanekile",
+        "ENTAILMENT_LABEL": "Ewe",
+        "NEUTRAL_LABEL": "Kananjalo",
+        "CONTRADICTION_LABEL": "Hayi"
+    },
+    "yor": {
+        "QUESTION_WORD": "Òótọ́",
+        "ENTAILMENT_LABEL": "Bẹ́ẹ̀ni",
+        "NEUTRAL_LABEL": "Àti pé",
+        "CONTRADICTION_LABEL": "Rárá"
+    },
+    "zul": {
+        "QUESTION_WORD": "Kulungile",
+        "ENTAILMENT_LABEL": "Yebo",
+        "NEUTRAL_LABEL": "Futhi",
+        "CONTRADICTION_LABEL": "Cha"
+    }
+}
+
+
+def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
+    """
+    Generate a yaml file for each language.
+
+    :param output_dir: The directory to output the files to.
+    :param overwrite: Whether to overwrite files if they already exist.
+    """
+    err = []
+    languages = ['eng', 'amh', 'ibo', 'fra', 'sna', 'wol', 'ewe', 'lin', 'lug', 'xho', 'kin', 'twi', 'zul', 'orm',
+                 'yor', 'hau', 'sot', 'swa']
+    for lang in languages:
+        try:
+            if mode == "native-direct":
+                QUESTION_WORD = LANGUAGES[lang]["QUESTION_WORD"]
+                ENTAILMENT_LABEL = LANGUAGES[lang]["ENTAILMENT_LABEL"]
+                NEUTRAL_LABEL = LANGUAGES[lang]["NEUTRAL_LABEL"]
+                CONTRADICTION_LABEL = LANGUAGES[lang]["CONTRADICTION_LABEL"]
+
+                file_name = f"afrixnli_native_direct_{lang}.yaml"
+                task_name = f"afrixnli_native_direct_{lang}"
+                yaml_template = "afrixnli_native_direct_yaml"
+                with open(
+                        f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8"
+                ) as f:
+                    f.write("# Generated by utils.py\n")
+                    yaml.dump(
+                        {
+                            "include": yaml_template,
+                            "task": task_name,
+                            "dataset_name": lang,
+                            "doc_to_choice": f"{{{{["
+                                           f"""premise+\", {QUESTION_WORD}? {ENTAILMENT_LABEL}, \"+hypothesis,"""
+                                           f"""premise+\", {QUESTION_WORD}? {NEUTRAL_LABEL}, \"+hypothesis,"""
+                                           f"""premise+\", {QUESTION_WORD}? {CONTRADICTION_LABEL}, \"+hypothesis"""
+                                           f"]}}}}",
+                        },
+                        f,
+                        allow_unicode=True,
+                    )
+            else:
+                file_name = f"afrixnli_{mode}_{lang}.yaml"
+                task_name = f"afrixnli_{mode}_{lang}"
+                yaml_template = f"afrixnli_{mode}_yaml"
+                with open(
+                        f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8"
+                ) as f:
+                    f.write("# Generated by utils.py\n")
+                    yaml.dump(
+                        {
+                            "include": yaml_template,
+                            "task": task_name,
+                            "dataset_name": lang
+                        },
+                        f,
+                        allow_unicode=True,
+                    )
+        except FileExistsError:
+            err.append(file_name)
+
+    if len(err) > 0:
+        raise FileExistsError(
+            "Files were not created because they already exist (use --overwrite flag):"
+            f" {', '.join(err)}"
+        )
+
+
+def main() -> None:
+    """Parse CLI args and generate language-specific yaml files."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--overwrite",
+        default=True,
+        action="store_true",
+        help="Overwrite files if they already exist",
+    )
+    parser.add_argument(
+        "--output-dir", default="./manual/translate", help="Directory to write yaml files to"
+    )
+    parser.add_argument(
+        "--mode",
+        default="manual_translate",
+        choices=["en_direct", "native-direct", "manual_direct", "manual_translate"],
+        help="Mode of chain-of-thought",
+    )
+    args = parser.parse_args()
+
+    gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode)
+
+
+if __name__ == "__main__":
+    main()