Merge pull request #18 from JessicaOjo/africamgsm

fix exact match bug and restructure mmlu folder

lm_eval/api/task.py

@@ -1367,11 +1367,18 @@ class ConfigurableTask(Task):
                             result_score = 0.0
                 else:
                     try:
-                        result_score = self._metric_fn_list[metric](
-                            references=[gold],
-                            predictions=[result],
-                            **self._metric_fn_kwargs[metric],
-                        )
+                        if metric == "exact_match":
+                            result_score = self._metric_fn_list[metric](
+                                references=[str(gold)],
+                                predictions=[str(result)],
+                                **self._metric_fn_kwargs[metric],
+                            )
+                        else:
+                            result_score = self._metric_fn_list[metric](
+                                references=[gold],
+                                predictions=[result],
+                                **self._metric_fn_kwargs[metric],
+                            )
                     except TypeError as error:  # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics
                         result_score = self._metric_fn_list[metric]([gold, result])
                     if isinstance(result_score, dict):

lm_eval/tasks/afrimgsm/gpt_direct.sh

@@ -4,7 +4,7 @@ models=(
   "gpt-3.5-turbo"
   "gpt-4-0125-preview"
 )
-task=afrimgsm_direct_eng,afrimgsm_direct_fra,afrimgsm_direct_swa #afrimgsm_direct_ewe,afrimgsm_direct_fra,afrimgsm_direct_hau,afrimgsm_direct_ibo,afrimgsm_direct_kin,afrimgsm_direct_lin,afrimgsm_direct_lug,afrimgsm_direct_orm,afrimgsm_direct_sna,afrimgsm_direct_sot,afrimgsm_direct_swa,afrimgsm_direct_twi,afrimgsm_direct_wol,afrimgsm_direct_xho,afrimgsm_direct_yor,afrimgsm_direct_zul
+task=afrimgsm_direct_amh,afrimgsm_direct_eng,afrimgsm_direct_ewe,afrimgsm_direct_fra,afrimgsm_direct_hau,afrimgsm_direct_ibo,afrimgsm_direct_kin,afrimgsm_direct_lin,afrimgsm_direct_lug,afrimgsm_direct_orm,afrimgsm_direct_sna,afrimgsm_direct_sot,afrimgsm_direct_swa,afrimgsm_direct_twi,afrimgsm_direct_wol,afrimgsm_direct_xho,afrimgsm_direct_yor,afrimgsm_direct_zul
 
 for model in "${models[@]}"
 do

lm_eval/tasks/afrimmlu/afrimmlu_amh.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml

 dataset_name: amh
 include: afrimmlu_common_yaml
-task: afrimmlu_amh
+task: afrimmlu_direct_amh

lm_eval/tasks/afrimmlu/afrimmlu_eng.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml

 dataset_name: eng
 include: afrimmlu_common_yaml
-task: afrimmlu_eng
+task: afrimmlu_direct_eng
 

lm_eval/tasks/afrimmlu/afrimmlu_ewe.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml

 dataset_name: ewe
 include: afrimmlu_common_yaml
-task: afrimmlu_ewe
+task: afrimmlu_direct_ewe
 

lm_eval/tasks/afrimmlu/afrimmlu_fra.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml

 dataset_name: fra
 include: afrimmlu_common_yaml
-task: afrimmlu_fra
\ No newline at end of file
+task: afrimmlu_direct_fra
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_hau.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml

 dataset_name: hau
 include: afrimmlu_common_yaml
-task: afrimmlu_hau
\ No newline at end of file
+task: afrimmlu_direct_hau
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_ibo.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml

 dataset_name: ibo
 include: afrimmlu_common_yaml
-task: afrimmlu_ibo
\ No newline at end of file
+task: afrimmlu_direct_ibo
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_kin.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml

 dataset_name: kin
 include: afrimmlu_common_yaml
-task: afrimmlu_kin
\ No newline at end of file
+task: afrimmlu_direct_kin
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_lin.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml

 dataset_name: lin
 include: afrimmlu_common_yaml
-task: afrimmlu_lin
\ No newline at end of file
+task: afrimmlu_direct_lin
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_lug.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml

 dataset_name: lug
 include: afrimmlu_common_yaml
-task: afrimmlu_lug
\ No newline at end of file
+task: afrimmlu_direct_lug
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_orm.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml

 dataset_name: orm
 include: afrimmlu_common_yaml
-task: afrimmlu_orm
\ No newline at end of file
+task: afrimmlu_direct_orm
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_sna.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml

 dataset_name: sna
 include: afrimmlu_common_yaml
-task: afrimmlu_sna
\ No newline at end of file
+task: afrimmlu_direct_sna
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_sot.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml

 dataset_name: sot
 include: afrimmlu_common_yaml
-task: afrimmlu_sot
\ No newline at end of file
+task: afrimmlu_direct_sot
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_swa.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml

 dataset_name: swa
 include: afrimmlu_common_yaml
-task: afrimmlu_swa
\ No newline at end of file
+task: afrimmlu_direct_swa
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_twi.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml

 dataset_name: twi
 include: afrimmlu_common_yaml
-task: afrimmlu_twi
\ No newline at end of file
+task: afrimmlu_direct_twi
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_wol.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml

 dataset_name: wol
 include: afrimmlu_common_yaml
-task: afrimmlu_wol
\ No newline at end of file
+task: afrimmlu_direct_wol
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_xho.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml

 dataset_name: xho
 include: afrimmlu_common_yaml
-task: afrimmlu_xho
\ No newline at end of file
+task: afrimmlu_direct_xho
\ No newline at end of file

lm_eval/tasks/afrimmlu/afrimmlu_yor.yaml → lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml

 dataset_name: yor
 include: afrimmlu_common_yaml
-task: afrimmlu_yor
\ No newline at end of file
+task: afrimmlu_direct_yor
\ No newline at end of file