Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
utils.py 1.18 KB
Newer Older
Jess's avatar
Irokobench: Benchmark Dataset for African languages (#2042)  
Jess committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43 
import re
import sys
import unicodedata

from sklearn.metrics import f1_score
from lm_eval.filters.extraction import RegexFilter


def doc_to_choice(doc):
    choices = eval(doc["choices"])
    return choices


def doc_to_text(doc):
    output = """You are a highly knowledgeable and intelligent artificial intelligence 
                model answers multiple-choice questions about {subject}
                
                Question: {question}

                Choices:
                        A: {choice1}
                        B: {choice2}
                        C: {choice3}
                        D: {choice4}
                       
                Answer:  """
    
    choices = eval(doc["choices"])
    text = output.format(subject=doc['subject'],
                         question=doc['question'],
                         choice1=choices[0],
                         choice2=choices[1],
                         choice3=choices[2],
                         choice4=choices[3])
    return text


def weighted_f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    fscore = f1_score(golds, preds, average="weighted")
    return fscore