cleanups and readmes

lm_eval/tasks/afrimgsm/direct_native/direct_native_yaml

-# This file will be included in the generated language-specific task configs.
-# It doesn't have a yaml file extension as it is not meant to be imported directly
-# by the harness.
-group: afrimgsm_direct
-dataset_path: masakhane/afrimgsm
-dataset_name: null  # Overridden by language-specific config.
-output_type: generate_until
-# training_split: train
-test_split: test
-target_delimiter: ""
-generation_kwargs:
-  until:
-    - "\n\n"
-    - "\n"
-  do_sample: false
-  temperature: 0.0
-filter_list:
-  - name: remove_whitespace
-    filter:
-      - function: remove_whitespace
-      - function: take_first
-  - filter:
-    - function: regex
-      group_select: -1
-      regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-    - function: take_first
-    name: flexible-extract
-metric_list:
-  - metric: exact_match
-    aggregation: mean
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: true
-metadata:
-  version: 2.0

lm_eval/tasks/afrimgsm/en_cot/cot_yaml

 # This file will be included in the generated language-specific task configs.
-# It doesn't have a yaml file extension as it is not meant to be imported directly
-# by the harness.
-group: afrimgsm_direct
+# It doesn't have a yaml file extension as it is not meant to be imported directly by the harness.
+group:
+    - mgsm
+    - afrimgsm
+    - afrimgsm_en_cot
 dataset_path: masakhane/afrimgsm
 dataset_name: null  # Overridden by language-specific config.
 output_type: generate_until

lm_eval/tasks/afrimgsm/gpt_direct.sh

-#!/bin/bash
-
-models=(
-  "gpt-3.5-turbo"
-  "gpt-4-0125-preview"
-)
-task=afrimgsm_direct_amh,afrimgsm_direct_eng,afrimgsm_direct_ewe,afrimgsm_direct_fra,afrimgsm_direct_hau,afrimgsm_direct_ibo,afrimgsm_direct_kin,afrimgsm_direct_lin,afrimgsm_direct_lug,afrimgsm_direct_orm,afrimgsm_direct_sna,afrimgsm_direct_sot,afrimgsm_direct_swa,afrimgsm_direct_twi,afrimgsm_direct_wol,afrimgsm_direct_xho,afrimgsm_direct_yor,afrimgsm_direct_zul
-
-for model in "${models[@]}"
-do
-  echo "Evaluating model: $model"
-  for fewshot in 0 2 4 6 8
-  do
-    export OUTPUT_DIR=results/$fewshot
-
-    mkdir -p "$OUTPUT_DIR"
-
-    lm_eval --model openai-chat-completions \
-            --model_args model="${model}" \
-            --tasks $task \
-            --output_path "$OUTPUT_DIR" \
-            --num_fewshot $fewshot \
-            --verbosity DEBUG
-  done
-done
\ No newline at end of file

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_amh.yaml

-# Generated by utils.py
-dataset_name: amh
-doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nበቅደም ተከተል መልስ:"}}{% else %}{{"ጥያቄ: "+question+"\nበቅደም ተከተል መልስ:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: መልሱ (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'ጥያቄ:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_amh

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_eng.yaml

-# Generated by utils.py
-dataset_name: eng
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: The answer is (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_eng

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_ewe.yaml

-# Generated by utils.py
-dataset_name: ewe
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: ŋuɖoɖoae nye (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_ewe

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_fra.yaml

-# Generated by utils.py
-dataset_name: fra
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: La réponse est(\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_fra

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_hau.yaml

-# Generated by utils.py
-dataset_name: hau
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Amsar ita ce (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_hau

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_ibo.yaml

-# Generated by utils.py
-dataset_name: ibo
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Azịza ya bụ (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_ibo

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_kin.yaml

-# Generated by utils.py
-dataset_name: kin
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Igisubizo ni (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_kin

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_lin.yaml

-# Generated by utils.py
-dataset_name: lin
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Eyano ezali (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_lin

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_lug.yaml

-# Generated by utils.py
-dataset_name: lug
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Ansa eri (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_lug

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_orm.yaml

-# Generated by utils.py
-dataset_name: orm
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Deebiin isaa (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_orm

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_sna.yaml

-# Generated by utils.py
-dataset_name: sna
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Mhinduro kumubvunzo ndi (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_sna

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_sot.yaml

-# Generated by utils.py
-dataset_name: sot
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Karabo ke (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_sot

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_swa.yaml

-# Generated by utils.py
-dataset_name: swa
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Jibu ni (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_swa

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_twi.yaml

-# Generated by utils.py
-dataset_name: twi
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Ne nnyiano yɛ (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_twi

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_wol.yaml

-# Generated by utils.py
-dataset_name: wol
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Tontu li (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_wol

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_xho.yaml

-# Generated by utils.py
-dataset_name: xho
-doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Impendulo ngu (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_xho

lm_eval/tasks/afrimgsm/native_cot/afrimgsm_native_cot_yor.yaml

-# Generated by utils.py
-dataset_name: yor
-doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nÌdáhùn lẹ́sẹsẹ:"}}{% else %}{{"Ìbéèrè: "+question+"\nÌdáhùn lẹ́sẹsẹ:"}}{% endif %}'
-filter_list:
-- filter:
-  - function: regex
-    regex_pattern: Ìdáhùn náà ni (\-?[0-9\.\,]+)
-  - function: take_first
-  name: strict-match
-- filter:
-  - function: regex
-    group_select: -1
-    regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+)
-  - function: take_first
-  name: flexible-extract
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Ìbéèrè:'
-  - </s>
-  - <|im_end|>
-include: cot_yaml
-task: afrimgsm_native_cot_yor