modefied mgsm

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml

 # Generated by utils.py
 dataset_name: amh
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_amh

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml

 # Generated by utils.py
 dataset_name: eng
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_eng

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml

 # Generated by utils.py
 dataset_name: ewe
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_ewe

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml

 # Generated by utils.py
 dataset_name: fra
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_fra

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml

 # Generated by utils.py
 dataset_name: hau
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_hau

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml

 # Generated by utils.py
 dataset_name: ibo
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_ibo

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml

 # Generated by utils.py
 dataset_name: kin
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_kin

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml

 # Generated by utils.py
 dataset_name: lin
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_lin

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml

 # Generated by utils.py
 dataset_name: lug
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_lug

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml

 # Generated by utils.py
 dataset_name: orm
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_orm

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml

 # Generated by utils.py
 dataset_name: sna
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_sna

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml

 # Generated by utils.py
 dataset_name: sot
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_sot

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml

 # Generated by utils.py
 dataset_name: swa
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_swa

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml

 # Generated by utils.py
 dataset_name: twi
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_twi

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml

 # Generated by utils.py
 dataset_name: wol
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_wol

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml

 # Generated by utils.py
 dataset_name: xho
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_xho

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml

 # Generated by utils.py
 dataset_name: yor
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_yor

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml

 # Generated by utils.py
 dataset_name: zul
-include: afrimgsm_common_yaml
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: direct_yaml
 task: afrimgsm_direct_zul

lm_eval/tasks/afrimgsm/direct/afrimgsm_common_yaml → lm_eval/tasks/afrimgsm/direct/direct_yaml

-group:
- - mgsm_direct
- - afrimgsm
+# This file will be included in the generated language-specific task configs.
+# It doesn't have a yaml file extension as it is not meant to be imported directly
+# by the harness.
+group: afrimgsm_direct
 dataset_path: masakhane/afrimgsm
+dataset_name: null  # Overridden by language-specific config.
 output_type: generate_until
 training_split: train
 test_split: test
-fewshot_split: train
 target_delimiter: ""
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
 generation_kwargs:
-  do_sample: false
   until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
+    - "\n\n"
+    - "\n"
+  do_sample: false
+  temperature: 0.0
 filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
   - filter:
     - function: regex
       group_select: -1
-      regex_pattern: (-?[0-9.,]{2,})|(-?[0-9]+)
+      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
     - function: take_first
     name: flexible-extract
-  - filter:
-    - function: regex-numbers
-      group_select: -1
-      regex_pattern: (\d{1,10}(?:,\d{3})*(?:[.,]\d{3})?)([^\d()]*)
-    - function: take_first
-    name: flexible-extract-new
 metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
-  - metric: f1
-    aggregation: !function utils.weighted_f1_score
-    average: weighted
-    higher_is_better: True
-    ignore_case: true
-    ignore_punctuation: true
-  - metric: squad
-    aggregation: squad_f1
-    average: weighted
-    higher_is_better: True
-    ignore_case: true
-    ignore_punctuation: true
 metadata:
   version: 2.0

lm_eval/tasks/afrimgsm/direct/directutils.py

+def doc_to_target(doc):
+    return doc['answer']