Merge pull request #870 from EleutherAI/lintangsutawika-patch-4

Create cot_yaml

lm_eval/tasks/mgsm/native_cot/cot_yaml

+# This file will be included in the generated language-specific task configs.
+# It doesn't have a yaml file extension as it is not meant to be imported directly
+# by the harness.
+group: mgsm_cot_native
+dataset_path: juletxara/mgsm
+dataset_name: null  # Overridden by language-specific config.
+output_type: greedy_until
+training_split: train
+test_split: test
+target_delimiter: ""
+generation_kwargs:
+  until:
+    - "\n\n"
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+filter_list:
+  - name: "get-answer"
+    filter:
+      - function: "regex"
+        regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_bn.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[16+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nধাপে ধাপে উত্তর:"}}{% else
   %}{{"প্রশ্ন: "+question+"\nধাপে ধাপে উত্তর:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_bn_direct
+task: mgsm_bn_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_de.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[28+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nSchritt-für-Schritt-Antwort:"}}{%
   else %}{{"Frage: "+question+"\nSchritt-für-Schritt-Antwort:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_de_direct
+task: mgsm_de_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_en.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[20+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else
   %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_en_direct
+task: mgsm_en_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_es.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[22+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta paso a paso:"}}{%
   else %}{{"Pregunta: "+question+"\nRespuesta paso a paso:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_es_direct
+task: mgsm_es_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_fr.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[25+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nRéponse étape par étape :"}}{%
   else %}{{"Question : "+question+"\nRéponse étape par étape :"}}{% endif %}'
 include: cot_yaml
-task: mgsm_fr_direct
+task: mgsm_fr_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ja.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[10+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題:
   "+question+"\nステップごとの答え:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_ja_direct
+task: mgsm_ja_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ru.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[17+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nПошаговоерешение:"}}{% else
   %}{{"Задача: "+question+"\nПошаговоерешение:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_ru_direct
+task: mgsm_ru_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_sw.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[24+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nJibu la Hatua kwa Hatua:"}}{%
   else %}{{"Swali: "+question+"\nJibu la Hatua kwa Hatua:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_sw_direct
+task: mgsm_sw_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_te.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[18+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nదశలవారీగా సమాధానం:"}}{% else
   %}{{"ప్రశ్న: "+question+"\nదశలవారీగా సమాధానం:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_te_direct
+task: mgsm_te_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_th.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[17+1]}}{% else %}{{answer_nu
 doc_to_text: '{% if answer is not none %}{{question+"\nคำตอบทีละขั้นตอน:"}}{% else
   %}{{"โจทย์: "+question+"\nคำตอบทีละขั้นตอน:"}}{% endif %}'
 include: cot_yaml
-task: mgsm_th_direct
+task: mgsm_th_native_cot

lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_zh.yaml

@@ -5,4 +5,4 @@ doc_to_target: '{% if answer is not none %}{{answer[5+1]}}{% else %}{{answer_num
 doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题: "+question+"\n逐步解答:"}}{%
   endif %}'
 include: cot_yaml
-task: mgsm_zh_direct
+task: mgsm_zh_native_cot