Merge pull request #12 from JessicaOjo/africamgsm

update f1 function

lm_eval/tasks/afrixnli/en-direct/afrixnli_en_direct_yaml

@@ -20,7 +20,7 @@ should_decontaminate: true
 doc_to_decontamination_query: premise
 metric_list:
   - metric: f1
-    aggregation: f1
+    aggregation: !function utils.weighted_f1_score
     average: weighted
     higher_is_better: True
     ignore_case: true
@@ -30,8 +30,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
-    regexes_to_ignore:
-      - ","
-      - "\\$"
 metadata:
   version: 1.0

lm_eval/tasks/afrixnli/native-direct/afrixnli_native_direct_yaml

@@ -11,7 +11,7 @@ doc_to_target: label
 doc_to_text: ""
 metric_list:
   - metric: f1
-    aggregation: f1
+    aggregation: !function utils.weighted_f1_score
     average: weighted
     higher_is_better: True
     ignore_case: true

lm_eval/tasks/afrixnli/native-direct/utils.py

+from sklearn.metrics import f1_score
+
+
+def weighted_f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = f1_score(golds, preds, average="weighted")
+    return fscore

lm_eval/tasks/afrixnli/utils.py

@@ -50,6 +50,12 @@ LANGUAGES = {
         "NEUTRAL_LABEL": "Na none",
         "CONTRADICTION_LABEL": "Oya"
     },
+    "lin": {
+        "QUESTION_WORD": "Malamu",
+        "ENTAILMENT_LABEL": "Iyo",
+        "NEUTRAL_LABEL": "Lisusu",
+        "CONTRADICTION_LABEL": "Te"
+    },
     "lug": {
         "QUESTION_WORD": "Kituufu",
         "ENTAILMENT_LABEL": "Yee",
@@ -121,7 +127,7 @@ def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
     :param overwrite: Whether to overwrite files if they already exist.
     """
     err = []
-    languages = ['eng', 'amh', 'ibo', 'fra', 'sna', 'wol', 'ewe', 'lug', 'xho', 'kin', 'twi', 'zul', 'orm',
+    languages = ['eng', 'amh', 'ibo', 'fra', 'sna', 'wol', 'ewe', 'lin', 'lug', 'xho', 'kin', 'twi', 'zul', 'orm',
                  'yor', 'hau', 'sot', 'swa']
     for lang in languages:
         try: