tidy up

lm_eval/tasks/mgsm/README.md

@@ -53,12 +53,32 @@ Homepage: https://github.com/google-research/url-nlp/tree/main/mgsm
 
 #### Groups
 
-* `mgsm`
-
-#### Tasks
-
-* `task_name`: `1-sentence description of what this particular task does`
-* `task_name2`: ...
+* `mgsm_direct`: Direct question
+  * `mgsm_direct_bn`: Bengali
+  * `mgsm_direct_de`: German
+  * `mgsm_direct_en`: English
+  * `mgsm_direct_es`: Spanish
+  * `mgsm_direct_fr`: French
+  * `mgsm_direct_ja`: Japanese
+  * `mgsm_direct_ru`: Russian
+  * `mgsm_direct_sw`: Swahili
+  * `mgsm_direct_te`: Telugu
+  * `mgsm_direct_th`: Thai
+  * `mgsm_direct_zh`: Chinese
+* `mgsm_cot_native`: Question with Answer followed by CoT prompt in the same language as the dataset.
+  * `mgsm_cot_native_bn`: Bengali
+  * `mgsm_cot_native_de`: German
+  * `mgsm_cot_native_en`: English
+  * `mgsm_cot_native_es`: Spanish
+  * `mgsm_cot_native_fr`: French
+  * `mgsm_cot_native_ja`: Japanese
+  * `mgsm_cot_native_ru`: Russian
+  * `mgsm_cot_native_sw`: Swahili
+  * `mgsm_cot_native_te`: Telugu
+  * `mgsm_cot_native_th`: Thai
+  * `mgsm_cot_native_zh`: Chinese
+
+Examplar Samples: https://github.com/google-research/url-nlp/blob/main/mgsm/exemplars.py
 
 ### Checklist
 

lm_eval/tasks/mgsm/direct_yaml → lm_eval/tasks/mgsm/direct/direct_yaml

 # This file will be included in the generated language-specific task configs.
 # It doesn't have a yaml file extension as it is not meant to be imported directly
 # by the harness.
-group: mgsm
+group: mgsm_direct
 dataset_path: juletxara/mgsm
 dataset_name: null  # Overridden by language-specific config.
 output_type: greedy_until

lm_eval/tasks/mgsm/mgsm_de_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[7+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAntwort"}}{% else %}{{"Frage:
   "+question+"\nAntwort"}}{% endif %}'
 include: direct_yaml
-task: mgsm_de_direct
+task: mgsm_direct_de

lm_eval/tasks/mgsm/mgsm_en_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Question:
   "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_en_direct
+task: mgsm_direct_en

lm_eval/tasks/mgsm/mgsm_es_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Pregunta:
   "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_es_direct
+task: mgsm_direct_es

lm_eval/tasks/mgsm/mgsm_fr_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Question
   : "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_fr_direct
+task: mgsm_direct_fr

lm_eval/tasks/mgsm/mgsm_ja_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"問題: "+question+"\nAnswer"}}{%
   endif %}'
 include: direct_yaml
-task: mgsm_ja_direct
+task: mgsm_direct_ja

lm_eval/tasks/mgsm/mgsm_ru_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Задача:
   "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_ru_direct
+task: mgsm_direct_ru

lm_eval/tasks/mgsm/mgsm_sw_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"Swali:
   "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_sw_direct
+task: mgsm_direct_sw

lm_eval/tasks/mgsm/mgsm_te_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"ప్రశ్న:
   "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_te_direct
+task: mgsm_direct_te

lm_eval/tasks/mgsm/mgsm_th_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"โจทย์:
   "+question+"\nAnswer"}}{% endif %}'
 include: direct_yaml
-task: mgsm_th_direct
+task: mgsm_direct_th

lm_eval/tasks/mgsm/mgsm_zh_direct.yaml → lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[6+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer"}}{% else %}{{"问题: "+question+"\nAnswer"}}{%
   endif %}'
 include: direct_yaml
-task: mgsm_zh_direct
+task: mgsm_direct_zh

lm_eval/tasks/mgsm/cot_yaml → lm_eval/tasks/mgsm/native_cot/cot_yaml

 # This file will be included in the generated language-specific task configs.
 # It doesn't have a yaml file extension as it is not meant to be imported directly
 # by the harness.
-group: mgsm
+group: mgsm_cot_native
 dataset_path: juletxara/mgsm
 dataset_name: null  # Overridden by language-specific config.
 output_type: greedy_until