Merge pull request #14 from JessicaOjo/afrimmlu

Afrimmlu

lm_eval/tasks/afrimmlu_translate/fewshot.sh

+lm_eval --model hf \
+        --model_args pretrained=masakhane/African-ultrachat-alpaca  \
+        --tasks afrimmlu_amh,afrimmlu_eng,afrimmlu_ewe,afrimmlu_fra,afrimmlu_hau,afrimmlu_ibo,afrimmlu_kin,afrimmlu_lin,afrimmlu_lug,afrimmlu_orm,afrimmlu_sna,afrimmlu_sot,afrimmlu_twi,afrimmlu_wol,afrimmlu_xho,afrimmlu_yor,afrimmlu_zul   \
+        --device cuda:0     \
+        --batch_size 1 \
+        --num_fewshot 0 \
+        --verbosity DEBUG \
+        --wandb_args project=afrimmlu
\ No newline at end of file

lm_eval/tasks/afrimmlu_translate/utils.py

+from sklearn.metrics import f1_score
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence 
+                model answers multiple-choice questions about '{subject}'
+                
+                Question: '''{question}'''
+
+                Choices:
+                        A: ''{choice1}'''
+                        B: ''{choice2}'''
+                        C: ''{choice3}'''
+                        D: ''{choice4}'''
+                       
+                Answer:  """
+    
+    choices = eval(doc["choices"])
+    text = output.format(subject=doc['subject'],
+                         question=doc['question'],
+                         choice1=choices[0],
+                         choice2=choices[1],
+                         choice3=choices[2],
+                         choice4=choices[3])
+    return text
+
+def weighted_f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = f1_score(golds, preds, average="weighted")
+    return fscore
\ No newline at end of file