other mgsm varieties

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_amh.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_amh.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_amh
+task: afrimgsm_direct_native_amh

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_eng.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_eng.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_eng
+task: afrimgsm_direct_native_eng

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_ewe.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_ewe.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_ewe
+task: afrimgsm_direct_native_ewe

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_fra.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_fra.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_fra
+task: afrimgsm_direct_native_fra

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_hau.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_hau.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_hau
+task: afrimgsm_direct_native_hau

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_ibo.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_ibo.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_ibo
+task: afrimgsm_direct_native_ibo

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_kin.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_kin.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_kin
+task: afrimgsm_direct_native_kin

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_lin.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_lin.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_lin
+task: afrimgsm_direct_native_lin

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_lug.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_lug.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_lug
+task: afrimgsm_direct_native_lug

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_orm.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_orm.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_orm
+task: afrimgsm_direct_native_orm

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_sna.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_sna.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_sna
+task: afrimgsm_direct_native_sna

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_sot.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_sot.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_sot
+task: afrimgsm_direct_native_sot

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_swa.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_swa.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_swa
+task: afrimgsm_direct_native_swa

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_twi.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_twi.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_twi
+task: afrimgsm_direct_native_twi

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_wol.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_wol.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_wol
+task: afrimgsm_direct_native_wol

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_xho.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_xho.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_xho
+task: afrimgsm_direct_native_xho

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_yor.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_yor.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_yor
+task: afrimgsm_direct_native_yor

lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_zul.yaml → lm_eval/tasks/afrimgsm/direct_native/afrimgsm_direct_native_zul.yaml

@@ -9,4 +9,4 @@ generation_kwargs:
   - </s>
   - <|im_end|>
 include: direct_native_yaml
-task: afrimgsm_direct_zul
+task: afrimgsm_direct_native_zul

lm_eval/tasks/afrimgsm/run.sh

 lm_eval --model hf   \
-        --model_args pretrained="google/gemma-7b"  --tasks afrimgsm_en_cot_eng,mgsm_en_cot_en,afrimgsm_native_cot_eng,mgsm_native_cot_en,afrimgsm_direct_eng,mgsm_direct_en,afrimgsm_direct_eng  \
+        --model_args pretrained="google/gemma-7b"  --tasks afrimgsm_en_cot_eng,mgsm_en_cot_en,afrimgsm_native_cot_eng,mgsm_native_cot_en,afrimgsm_direct_eng,mgsm_direct_en,afrimgsm_direct_native_eng  \
         --device cuda:0     \
         --batch_size 1  \
         --verbosity DEBUG \

lm_eval/tasks/afrimgsm/utils.py

@@ -105,7 +105,7 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
                 ANSWER = LANGUAGES[lang]["DIRECT"]
                 QUESTION = LANGUAGES[lang]["QUESTION"]
                 REGEX = None
-                task_name = f"afrimgsm_direct_{lang}"
+                task_name = f"afrimgsm_direct_native_{lang}"
                 yaml_template = "direct_native_yaml"
             elif mode == "native-cot":
                 ANSWER = LANGUAGES[lang]["ANSWER"]