Merge pull request #24 from JessicaOjo/africamgsm

xnli changes for open models

lm_eval/tasks/afrixnli/manual/direct/afrixnli_manual_direct_yaml

@@ -4,7 +4,7 @@ group:
     - afrixnli-manual
 dataset_path: masakhane/afrixnli
 dataset_name: null
-output_type: multiple_choice_gpt
+output_type: multiple_choice
 validation_split: validation
 test_split: test
 fewshot_split: validation
@@ -24,7 +24,7 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
   - metric: acc
-    aggregation: acc_gpt
+    aggregation: mean
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true

lm_eval/tasks/afrixnli/manual/translate/afrixnli_manual_translate_yaml

@@ -5,7 +5,7 @@ group:
     - afrixnli-translate-test
 dataset_path: masakhane/afrixnli-translate-test
 dataset_name: null
-output_type: multiple_choice_gpt
+output_type: multiple_choice
 test_split: test
 doc_to_text: !function utils.doc_to_text
 doc_to_target: !function utils.doc_to_target
@@ -23,7 +23,7 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
   - metric: acc
-    aggregation: acc_gpt
+    aggregation: mean
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true