add afrimgsm -direct

afrimgsm_direct_eng

+!!@@##@@!! -- Example 0
+If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?
+Answer:5
+
+Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?
+Answer:
+!!@@##@@!! -- Example 1
+Olivia has $23. She bought five bagels for $3 each. How much money does she have left?
+Answer:8
+
+A robe takes 2 bolts of blue fiber and half that much white fiber.  How many bolts in total does it take?
+Answer:
+!!@@##@@!! -- Example 2
+There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?
+Answer:29
+
+Josh decides to try flipping a house.  He buys a house for $80,000 and then puts in $50,000 in repairs.  This increased the value of the house by 150%.  How much profit did he make?
+Answer:

lm_eval/tasks/afrimgsm/direct.sh

+#!/bin/bash
+
+models=(
+  "masakhane/African-ultrachat-alpaca"
+  "masakhane/zephyr-7b-gemma-sft-african-alpaca"
+  "masakhane/zephyr-7b-gemma-sft-african-ultrachat-5k"
+  "google/flan-t5-xxl"
+  "bigscience/mt0-xxl-mt"
+  "CohereForAI/aya-101"
+  "bigscience/bloomz-7b1-mt"
+  "meta-llama/Llama-2-7b-chat-hf"
+  "meta-llama/Meta-Llama-3-8B-Instruct"
+  "meta-llama/Meta-Llama-3-70B-Instruct"
+  "google/gemma-1.1-7b-it"
+  "RWKV/v5-EagleX-v2-7B-HF"
+  "RWKV/rwkv-6-world-7b"
+)
+
+for model in "${models[@]}"
+do
+  echo "Evaluating model: $model"
+  for task in afrimgsm_direct_eng afrimgsm_direct_amh afrimgsm_direct_ibo afrimgsm_direct_fra afrimgsm_direct_sna afrimgsm_direct_lin afrimgsm_direct_wol afrimgsm_direct_ewe afrimgsm_direct_lug afrimgsm_direct_xho afrimgsm_direct_kin afrimgsm_direct_twi afrimgsm_direct_zul afrimgsm_direct_orm afrimgsm_direct_yor afrimgsm_direct_hau afrimgsm_direct_sot afrimgsm_direct_swa
+  do
+    export OUTPUT_DIR=results/${model##*/}/${task}
+
+    mkdir -p "$OUTPUT_DIR"
+
+    lm_eval --model hf \
+            --model_args "pretrained=${model}" \
+            --tasks task\
+            --device cuda:0     \
+            --batch_size 16 \
+            --num_fewshot 0 \
+            --verbosity DEBUG
+  done
+done
\ No newline at end of file

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml

+# Generated by utils.py
+dataset_name: amh
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_amh

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml

+# Generated by utils.py
+dataset_name: eng
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_eng

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml

+# Generated by utils.py
+dataset_name: ewe
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_ewe

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml

+# Generated by utils.py
+dataset_name: fra
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_fra

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml

+# Generated by utils.py
+dataset_name: hau
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_hau

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml

+# Generated by utils.py
+dataset_name: ibo
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_ibo

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml

+# Generated by utils.py
+dataset_name: kin
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_kin

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml

+# Generated by utils.py
+dataset_name: lin
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_lin

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml

+# Generated by utils.py
+dataset_name: lug
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_lug

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml

+# Generated by utils.py
+dataset_name: orm
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_orm

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml

+# Generated by utils.py
+dataset_name: sna
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_sna

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml

+# Generated by utils.py
+dataset_name: sot
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_sot

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml

+# Generated by utils.py
+dataset_name: swa
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_swa

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml

+# Generated by utils.py
+dataset_name: twi
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_twi

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml

+# Generated by utils.py
+dataset_name: wol
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_wol

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml

+# Generated by utils.py
+dataset_name: xho
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_xho

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml

+# Generated by utils.py
+dataset_name: yor
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_yor

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml

+# Generated by utils.py
+dataset_name: zul
+doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
+generation_kwargs:
+  do_sample: false
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
+include: afrimgsm_common_yaml
+task: afrimgsm_direct_zul