add mgsm yamls

lm_eval/tasks/mgsm/common_template_yaml

@@ -14,6 +14,12 @@ generation_kwargs:
     - "\n"
   do_sample: false
   temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+target_delimiter: " "
 metric_list:
   - metric: exact_match
     aggregation: mean

lm_eval/tasks/mgsm/mgsm_bn.yaml

+# Generated by utils.py
+dataset_name: bn
+doc_to_target: '{% if answer is not none %}{{answer[16+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nধাপে ধাপে উত্তর:"}}{% else
+  %}{{"প্রশ্ন: "+question+"\nধাপে ধাপে উত্তর:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_bn

lm_eval/tasks/mgsm/mgsm_de.yaml

+# Generated by utils.py
+dataset_name: de
+doc_to_target: '{% if answer is not none %}{{answer[28+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nSchritt-für-Schritt-Antwort:"}}{%
+  else %}{{"Frage: "+question+"\nSchritt-für-Schritt-Antwort:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_de

lm_eval/tasks/mgsm/mgsm_en.yaml

+# Generated by utils.py
+dataset_name: en
+doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else
+  %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_en

lm_eval/tasks/mgsm/mgsm_es.yaml

+# Generated by utils.py
+dataset_name: es
+doc_to_target: '{% if answer is not none %}{{answer[22+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta paso a paso:"}}{%
+  else %}{{"Pregunta: "+question+"\nRespuesta paso a paso:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_es

lm_eval/tasks/mgsm/mgsm_fr.yaml

+# Generated by utils.py
+dataset_name: fr
+doc_to_target: '{% if answer is not none %}{{answer[25+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nRéponse étape par étape :"}}{%
+  else %}{{"Question : "+question+"\nRéponse étape par étape :"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_fr

lm_eval/tasks/mgsm/mgsm_ja.yaml

+# Generated by utils.py
+dataset_name: ja
+doc_to_target: '{% if answer is not none %}{{answer[10+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題:
+  "+question+"\nステップごとの答え:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_ja

lm_eval/tasks/mgsm/mgsm_ru.yaml

+# Generated by utils.py
+dataset_name: ru
+doc_to_target: '{% if answer is not none %}{{answer[17+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nПошаговоерешение:"}}{% else
+  %}{{"Задача: "+question+"\nПошаговоерешение:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_ru

lm_eval/tasks/mgsm/mgsm_sw.yaml

+# Generated by utils.py
+dataset_name: sw
+doc_to_target: '{% if answer is not none %}{{answer[24+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nJibu la Hatua kwa Hatua:"}}{%
+  else %}{{"Swali: "+question+"\nJibu la Hatua kwa Hatua:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_sw

lm_eval/tasks/mgsm/mgsm_te.yaml

+# Generated by utils.py
+dataset_name: te
+doc_to_target: '{% if answer is not none %}{{answer[18+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nదశలవారీగా సమాధానం:"}}{% else
+  %}{{"ప్రశ్న: "+question+"\nదశలవారీగా సమాధానం:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_te

lm_eval/tasks/mgsm/mgsm_th.yaml

+# Generated by utils.py
+dataset_name: th
+doc_to_target: '{% if answer is not none %}{{answer[17+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nคำตอบทีละขั้นตอน:"}}{% else
+  %}{{"โจทย์: "+question+"\nคำตอบทีละขั้นตอน:"}}{% endif %}'
+include: common_template_yaml
+task: mgsm_th

lm_eval/tasks/mgsm/mgsm_zh.yaml

+# Generated by utils.py
+dataset_name: zh
+doc_to_target: '{% if answer is not none %}{{answer[5+1]}}{% else %}{{answer_number|string}}{%
+  endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题: "+question+"\n逐步解答:"}}{%
+  endif %}'
+include: common_template_yaml
+task: mgsm_zh

lm_eval/tasks/mgsm/utils.py

-import argparse
-from typing import Dict, List
-
 import yaml
+import argparse
 
 
 LANGUAGES = {
@@ -51,18 +49,6 @@ LANGUAGES = {
     },
 }
 
-def doc_to_text(doc, QUESTION, ANSWER):
-    if doc["answer"] is not None:
-        return doc["question"] + "\n" + ANSWER
-    else:
-        return QUESTION + " " + doc["question"] + "\n" + ANSWER
-
-def doc_to_target(doc, QUESTION, ANSWER):
-    if doc["answer"] is not None:
-        return " " + doc["answer"][len(ANSWER) + 1 :]
-    else:
-        return " " + str(doc["answer_number"])
-
 
 def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
     """
@@ -86,8 +72,16 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None:
                         "include": "common_template_yaml",
                         "dataset_name": lang,
                         "task": f"mgsm_{lang}",
-                        "doc_to_text": doc_to_text(doc, QUESTION, ANSWER),
-                        "doc_to_target": doc_to_target(doc, QUESTION, ANSWER),
+                        "doc_to_text": f"""{{% if answer is not none %}}""" \
+                                       f"""{{{{question+"\\n{ANSWER}"}}}}""" \
+                                       f"""{{% else %}}""" \
+                                       f"""{{{{"{QUESTION} "+question+"\\n{ANSWER}"}}}}""" \
+                                       f"""{{% endif %}}""",
+                        "doc_to_target": f"""{{% if answer is not none %}}""" \
+                                         f"""{{{{answer[{len(ANSWER)}+1]}}}}""" \
+                                         f"""{{% else %}}""" \
+                                         f"""{{{{answer_number|string}}}}""" \
+                                         f"""{{% endif %}}""",
                     },
                     f,
                     allow_unicode=True,