Merge pull request #21 from JessicaOjo/afri_mgsm

Afri mgsm

lm_eval/tasks/afrimgsm/native_cot/cot_yaml

+# This file will be included in the generated language-specific task configs.
+# It doesn't have a yaml file extension as it is not meant to be imported directly
+# by the harness.
+group: afrimgsm_direct
+dataset_path: masakhane/afrimgsm
+dataset_name: null  # Overridden by language-specific config.
+output_type: generate_until
+training_split: train
+test_split: test
+generation_kwargs:
+  until:
+    - "\n\n"
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+filter_list:
+  - name: "get-answer"
+    filter:
+      - function: "regex"
+        regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+metadata:
+  version: 3.0

lm_eval/tasks/afrimgsm/run.sh

+lm_eval --model hf   \
+        --model_args pretrained="google/gemma-7b"  --tasks afrimgsm_en_cot_eng,mgsm_en_cot_en,afrimgsm_native_cot_eng,mgsm_native_cot_en,afrimgsm_direct_eng,mgsm_direct_en,afrimgsm_direct_native_eng  \
+        --device cuda:0     \
+        --batch_size 1  \
+        --verbosity DEBUG \
+        --limit 5
\ No newline at end of file

lm_eval/tasks/afrimgsm/utils.py

@@ -3,14 +3,48 @@ import argparse
 import yaml
 languages = ['eng', 'amh', 'ibo', 'fra', 'sna', 'lin', 'wol', 'ewe', 'lug', 'xho', 'kin', 'twi', 'zul', 'orm', 'yor', 'hau', 'sot', 'swa']
 
+languages_REGEX = {"eng":"The answer is (\\-?[0-9\\.\\,]+)",
+                   "amh":"መልሱ (\\-?[0-9\\.\\,]+)",
+                   "ibo":"Azịza ya bụ (\\-?[0-9\\.\\,]+)",
+                   'fra':"La réponse est(\\-?[0-9\\.\\,]+)",
+                    'sna':"Mhinduro kumubvunzo ndi (\\-?[0-9\\.\\,]+)",
+                    'lin':"Eyano ezali (\\-?[0-9\\.\\,]+)",
+                    'wol': "Tontu li (\\-?[0-9\\.\\,]+)",
+                    'ewe': "ŋuɖoɖoae nye (\\-?[0-9\\.\\,]+)",
+                    'lug': "Ansa eri (\\-?[0-9\\.\\,]+)", 
+                    'xho': "Impendulo ngu (\\-?[0-9\\.\\,]+)", 
+                    'kin': "Igisubizo ni (\\-?[0-9\\.\\,]+)", 
+                    'twi': "Ne nnyiano yɛ (\\-?[0-9\\.\\,]+)", 
+                    'zul': "Impendulo ithi (\\-?[0-9\\.\\,]+)", 
+                    'orm': "Deebiin isaa (\\-?[0-9\\.\\,]+)", 
+                    'yor': "Ìdáhùn náà ni (\\-?[0-9\\.\\,]+)", 
+                    'hau': "Amsar ita ce (\\-?[0-9\\.\\,]+)", 
+                    'sot': "Karabo ke (\\-?[0-9\\.\\,]+)", 
+                    'swa': "Jibu ni (\\-?[0-9\\.\\,]+)",
+                    }  
+
 LANGUAGES = {}
 
 for lang in languages:
-    LANGUAGES[lang] = {  # English
-        "QUESTION": "Question:",
-        "ANSWER": "Step-by-Step Answer:",
-        "DIRECT": "Answer:",
-        "REGEX": "The answer is (\\-?[0-9\\.\\,]+)"} 
+    if lang == 'amh':
+         LANGUAGES[lang] = {  # English
+            "QUESTION": "ጥያቄ:",
+            "ANSWER": "በቅደም ተከተል መልስ:",
+            "DIRECT": "Answer:",
+            "REGEX": languages_REGEX[lang]} 
+    elif lang == 'yor':
+         LANGUAGES[lang] = {  # English
+            "QUESTION": "Ìbéèrè:",
+            "ANSWER": "Ìdáhùn lẹ́sẹsẹ:",
+            "DIRECT": "Answer:",
+            "REGEX": languages_REGEX[lang]} 
+        
+    else:
+        LANGUAGES[lang] = {  # English
+            "QUESTION": "Question:",
+            "ANSWER": "Step-by-Step Answer:",
+            "DIRECT": "Answer:",
+            "REGEX": languages_REGEX[lang]} 
 
 
 def add_regex_pattern(regex_pattern):
@@ -57,26 +91,35 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
     err = []
     for lang in LANGUAGES.keys():
         try:
-            QUESTION = LANGUAGES[lang]["QUESTION"]
 
             yaml_template = "cot_yaml"
             filter_list = {}
             DELIMITER = None
             if mode == "direct":
-                ANSWER = LANGUAGES[lang]["DIRECT"]
+                ANSWER = LANGUAGES['eng']["DIRECT"]
+                QUESTION = LANGUAGES['eng']["QUESTION"]
                 REGEX = None
                 task_name = f"afrimgsm_direct_{lang}"
                 yaml_template = "direct_yaml"
+            if mode == "direct-native":
+                ANSWER = LANGUAGES[lang]["DIRECT"]
+                QUESTION = LANGUAGES[lang]["QUESTION"]
+                REGEX = None
+                task_name = f"afrimgsm_direct_native_{lang}"
+                yaml_template = "direct_native_yaml"
             elif mode == "native-cot":
                 ANSWER = LANGUAGES[lang]["ANSWER"]
                 REGEX = LANGUAGES[lang]["REGEX"]
+                QUESTION = LANGUAGES[lang]["QUESTION"]
                 task_name = f"afrimgsm_native_cot_{lang}"
                 filter_list = add_regex_pattern(REGEX)
                 DELIMITER = "" if lang in ["zh", "ja"] else None
             elif mode == "en-cot":
-                ANSWER = LANGUAGES["en"]["ANSWER"]
-                REGEX = LANGUAGES["en"]["REGEX"]
+                ANSWER = LANGUAGES["eng"]["ANSWER"]
+                REGEX = LANGUAGES["eng"]["REGEX"]
+                QUESTION = LANGUAGES["eng"]["QUESTION"]
                 task_name = f"afrimgsm_en_cot_{lang}"
+            
 
             file_name = f"{task_name}.yaml"
             ANSWER_TO_SKIP = len(LANGUAGES[lang]["ANSWER"]) + 1
@@ -135,7 +178,7 @@ def main() -> None:
     parser.add_argument(
         "--mode",
         default="native-cot",
-        choices=["direct", "native-cot", "en-cot"],
+        choices=["direct","direct-native", "native-cot", "en-cot"],
         help="Mode of chain-of-thought",
     )
     args = parser.parse_args()