add utility

lm_eval/tasks/afrimmlu/direct/utils.py

+from sklearn.metrics import f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence 
+                model answers multiple-choice questions about '{subject}'
+                
+                Question: '''{question}'''
+
+                Choices:
+                        A: ''{choice1}'''
+                        B: ''{choice2}'''
+                        C: ''{choice3}'''
+                        D: ''{choice4}'''
+                       
+                Answer:  """
+    
+    choices = eval(doc["choices"])
+    text = output.format(subject=doc['subject'],
+                         question=doc['question'],
+                         choice1=choices[0],
+                         choice2=choices[1],
+                         choice3=choices[2],
+                         choice4=choices[3])
+    return text
+
+
+def weighted_f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = f1_score(golds, preds, average="weighted")
+    return fscore
\ No newline at end of file

lm_eval/tasks/afrimmlu/translate/utils.py

+from sklearn.metrics import f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence 
+                model answers multiple-choice questions about '{subject}'
+                
+                Question: '''{question}'''
+
+                Choices:
+                        A: ''{choice1}'''
+                        B: ''{choice2}'''
+                        C: ''{choice3}'''
+                        D: ''{choice4}'''
+                       
+                Answer:  """
+    
+    choices = eval(doc["choices"])
+    text = output.format(subject=doc['subject'],
+                         question=doc['question'],
+                         choice1=choices[0],
+                         choice2=choices[1],
+                         choice3=choices[2],
+                         choice4=choices[3])
+    return text
+
+
+def weighted_f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = f1_score(golds, preds, average="weighted")
+    return fscore
\ No newline at end of file