restructure mmlu folder

lm_eval/tasks/afrimmlu/afrimmlu_amh.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml

 dataset_name: amh
 include: afrimmlu_common_yaml
-task: afrimmlu_amh
+task: afrimmlu_direct_amh

lm_eval/tasks/afrimmlu/afrimmlu_eng.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml

 dataset_name: eng
 include: afrimmlu_common_yaml
-task: afrimmlu_eng
+task: afrimmlu_direct_eng
 

lm_eval/tasks/afrimmlu/afrimmlu_ewe.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml

 dataset_name: ewe
 include: afrimmlu_common_yaml
-task: afrimmlu_ewe
+task: afrimmlu_direct_ewe
 

lm_eval/tasks/afrimmlu/afrimmlu_fra.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml

 dataset_name: fra
 include: afrimmlu_common_yaml
-task: afrimmlu_fra
\ No newline at end of file
+task: afrimmlu_direct_fra
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_hau.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml

 dataset_name: hau
 include: afrimmlu_common_yaml
-task: afrimmlu_hau
\ No newline at end of file
+task: afrimmlu_direct_hau
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_ibo.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml

 dataset_name: ibo
 include: afrimmlu_common_yaml
-task: afrimmlu_ibo
\ No newline at end of file
+task: afrimmlu_direct_ibo
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_kin.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml

 dataset_name: kin
 include: afrimmlu_common_yaml
-task: afrimmlu_kin
\ No newline at end of file
+task: afrimmlu_direct_kin
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_lin.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml

 dataset_name: lin
 include: afrimmlu_common_yaml
-task: afrimmlu_lin
\ No newline at end of file
+task: afrimmlu_direct_lin
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_lug.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml

 dataset_name: lug
 include: afrimmlu_common_yaml
-task: afrimmlu_lug
\ No newline at end of file
+task: afrimmlu_direct_lug
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_orm.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml

 dataset_name: orm
 include: afrimmlu_common_yaml
-task: afrimmlu_orm
\ No newline at end of file
+task: afrimmlu_direct_orm
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_sna.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml

 dataset_name: sna
 include: afrimmlu_common_yaml
-task: afrimmlu_sna
\ No newline at end of file
+task: afrimmlu_direct_sna
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_sot.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml

 dataset_name: sot
 include: afrimmlu_common_yaml
-task: afrimmlu_sot
\ No newline at end of file
+task: afrimmlu_direct_sot
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_swa.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml

 dataset_name: swa
 include: afrimmlu_common_yaml
-task: afrimmlu_swa
\ No newline at end of file
+task: afrimmlu_direct_swa
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_twi.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml

 dataset_name: twi
 include: afrimmlu_common_yaml
-task: afrimmlu_twi
\ No newline at end of file
+task: afrimmlu_direct_twi
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_wol.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml

 dataset_name: wol
 include: afrimmlu_common_yaml
-task: afrimmlu_wol
\ No newline at end of file
+task: afrimmlu_direct_wol
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_xho.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml

 dataset_name: xho
 include: afrimmlu_common_yaml
-task: afrimmlu_xho
\ No newline at end of file
+task: afrimmlu_direct_xho
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_yor.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml

 dataset_name: yor
 include: afrimmlu_common_yaml
-task: afrimmlu_yor
\ No newline at end of file
+task: afrimmlu_direct_yor
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_zul.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml

 dataset_name: zul
 include: afrimmlu_common_yaml
-task: afrimmlu_zul
\ No newline at end of file
+task: afrimmlu_direct_zul
\ No newline at end of file

lm_eval/tasks/afrimmlu/fewshot.sh

 lm_eval --model hf \
         --model_args pretrained=masakhane/African-ultrachat-alpaca  \
-        --tasks afrimmlu_amh,afrimmlu_eng,afrimmlu_ewe,afrimmlu_fra,afrimmlu_hau,afrimmlu_ibo,afrimmlu_kin,afrimmlu_lin,afrimmlu_lug,afrimmlu_orm,afrimmlu_sna,afrimmlu_sot,afrimmlu_twi,afrimmlu_wol,afrimmlu_xho,afrimmlu_yor,afrimmlu_zul   \
+        --tasks afrimmlu_direct_amh,afrimmlu_direct_eng,afrimmlu_direct_ewe,afrimmlu_direct_fra,afrimmlu_direct_hau,afrimmlu_direct_ibo,afrimmlu_direct_kin,afrimmlu_direct_lin,afrimmlu_direct_lug,afrimmlu_direct_orm,afrimmlu_direct_sna,afrimmlu_direct_sot,afrimmlu_direct_twi,afrimmlu_direct_wol,afrimmlu_direct_xho,afrimmlu_direct_yor,afrimmlu_direct_zul   \
         --device cuda:0     \
         --batch_size 1 \
         --num_fewshot 0 \