Merge pull request #27 from JessicaOjo/africamgsm

add manual accuracy scores

lm_eval/tasks/afrixnli/manual/direct/afrixnli_manual_direct_yaml

@@ -32,7 +32,7 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
   - metric: acc
-    aggregation: mean
+    aggregation: !function utils.manual_accuracy_score
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true

lm_eval/tasks/afrixnli/manual/direct/utils.py

-from sklearn.metrics import f1_score
+from sklearn.metrics import f1_score, accuracy_score
 
 
 def doc_to_text(doc):
@@ -30,3 +30,12 @@ def weighted_f1_score(items):
     preds = unzipped_list[1]
     fscore = f1_score(golds, preds, average="weighted")
     return fscore
+
+
+def manual_accuracy_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+
+    accuracy = accuracy_score(golds, preds)
+    return accuracy

lm_eval/tasks/afrixnli/manual/translate/afrixnli_manual_translate_yaml

@@ -25,13 +25,13 @@ filter_list:
                 "neutral": ['neutral']}
 metric_list:
   - metric: f1
-    aggregation: !function utils.weighted_f1_score
+    aggregation:
     average: weighted
     higher_is_better: True
     ignore_case: true
     ignore_punctuation: true
   - metric: acc
-    aggregation: mean
+    aggregation: !function utils.manual_accuracy_score
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true

lm_eval/tasks/afrixnli/manual/translate/utils.py

-from sklearn.metrics import f1_score
+from sklearn.metrics import f1_score, accuracy_score
 
 
 def doc_to_text(doc):
@@ -30,3 +30,12 @@ def weighted_f1_score(items):
     preds = unzipped_list[1]
     fscore = f1_score(golds, preds, average="weighted")
     return fscore
+
+
+def manual_accuracy_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+
+    accuracy = accuracy_score(golds, preds)
+    return accuracy