update parsing logic of mgsm following gsm8k (#1462)

lm_eval/tasks/mgsm/direct/direct_yaml

@@ -19,6 +19,12 @@ filter_list:
     filter:
       - function: remove_whitespace
       - function: take_first
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -26,4 +32,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml

@@ -2,5 +2,11 @@
 dataset_name: bn
 doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"প্রশ্ন: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'প্রশ্ন:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_bn

lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml

@@ -2,5 +2,11 @@
 dataset_name: de
 doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAntwort:"}}{% else %}{{"Frage: "+question+"\nAntwort:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Frage:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_de

lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml

@@ -2,5 +2,11 @@
 dataset_name: en
 doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_en

lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml

@@ -2,5 +2,11 @@
 dataset_name: es
 doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Pregunta:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_es

lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml

@@ -2,5 +2,11 @@
 dataset_name: fr
 doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nRéponse :"}}{% else %}{{"Question : "+question+"\nRéponse :"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question :'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_fr

lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml

@@ -2,5 +2,11 @@
 dataset_name: ja
 doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '問題:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_ja

lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml

@@ -2,5 +2,11 @@
 dataset_name: ru
 doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Задача: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Задача:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_ru

lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml

@@ -2,5 +2,11 @@
 dataset_name: sw
 doc_to_target: '{% if answer is not none %}{{answer[25:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Swali: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Swali:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_sw

lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml

@@ -2,5 +2,11 @@
 dataset_name: te
 doc_to_target: '{% if answer is not none %}{{answer[19:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"ప్రశ్న: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'ప్రశ్న:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_te

lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml

@@ -2,5 +2,11 @@
 dataset_name: th
 doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"โจทย์: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'โจทย์:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_th

lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml

@@ -2,5 +2,11 @@
 dataset_name: zh
 doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '问题:'
+  - </s>
+  - <|im_end|>
 include: direct_yaml
 task: mgsm_direct_zh

lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -21,10 +21,16 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 filter_list:
-  - name: "get-answer"
+  - name: "strict-match"
     filter:
       - function: "regex"
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
+  - filter:
+    - function: regex
+      group_select: -1
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
+    - function: take_first
+    name: flexible-extract
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_bn.yaml

@@ -2,5 +2,11 @@
 dataset_name: bn
 doc_to_target: '{% if answer is not none %}{{answer[17:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"প্রশ্ন: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'প্রশ্ন:'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_bn

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_de.yaml

@@ -2,5 +2,11 @@
 dataset_name: de
 doc_to_target: '{% if answer is not none %}{{answer[29:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Frage: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Frage:'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_de

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_en.yaml

@@ -2,5 +2,11 @@
 dataset_name: en
 doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_en

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_es.yaml

@@ -2,5 +2,11 @@
 dataset_name: es
 doc_to_target: '{% if answer is not none %}{{answer[23:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Pregunta: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Pregunta:'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_es

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_fr.yaml

@@ -2,5 +2,11 @@
 dataset_name: fr
 doc_to_target: '{% if answer is not none %}{{answer[26:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question : "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question :'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_fr

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml

@@ -2,5 +2,11 @@
 dataset_name: ja
 doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - '問題:'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_ja

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ru.yaml

@@ -2,5 +2,11 @@
 dataset_name: ru
 doc_to_target: '{% if answer is not none %}{{answer[18:]}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Задача: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Задача:'
+  - </s>
+  - <|im_end|>
 include: cot_yaml
 task: mgsm_en_cot_ru