metrics.py 6.1 KB
Newer Older
&'s avatar
& committed
1
import math
&'s avatar
& committed
2
from collections import Iterable
&'s avatar
& committed
3
from pprint import pprint
&'s avatar
& committed
4
5
6
7

import numpy as np
import sacrebleu
import sklearn
Leo Gao's avatar
Leo Gao committed
8
import random
&'s avatar
& committed
9
10
11
12
13
14


def mean(arr):
    return sum(arr) / len(arr)


Leo Gao's avatar
Leo Gao committed
15
16
17
18
19
20
21
22
23
24
def stddev(arr):
    mu = mean(arr)
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))


def mean_stderr(arr):
    print(stddev(arr), len(arr))
    return stddev(arr) / math.sqrt(len(arr))


&'s avatar
& committed
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def median(arr):
    return arr[len(arr) // 2]


def matthews_corrcoef(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    return sklearn.metrics.matthews_corrcoef(golds, preds)


def f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    fscore = sklearn.metrics.f1_score(golds, preds)

    return np.max(fscore)


def acc_all(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
    return acc

Leo Gao's avatar
Leo Gao committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def acc_all_stderr(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
    return acc

&'s avatar
& committed
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


def perplexity(items):
    return math.exp(-mean(items))


def bleu(items):
    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
    for evaluating a generated sentence to a reference sentence. It counts matching
    n-grams in the candidate translation to n-grams in the reference text, where
    1-gram or unigram would be each token and a bigram comparison would be each
    word pair. The comparison is made regardless of word order
    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
    Paper: https://www.aclweb.org/anthology/P02-1040/

    Higher is better
    """
&'s avatar
metrics  
& committed
104
105
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
106
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
107
108
    return sacrebleu.corpus_bleu(preds, refs).score

&'s avatar
& committed
109
110
111
112
113
114
115
116
117

def chrf(items):
    """chrF++ is a tool for automatic evaluation of machine translation output
    based on character n-gram precision and recall enhanced with word n-grams.
    Source: https://github.com/m-popovic/chrF
    Paper: https://www.aclweb.org/anthology/W15-3049.pdf

    Higher is better  # TODO I think
    """
&'s avatar
metrics  
& committed
118
119
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
120
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
121
122
    return sacrebleu.corpus_chrf(preds, refs).score

&'s avatar
& committed
123
124
125
126
127
128
129
130
131
132

def ter(items):
    """Translation Error Rate is an error metric for machine translation that
    measures the number of edits required to change a system output into one
    of the references
    Source: http://www.cs.umd.edu/~snover/tercom/
    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf

    Lower is better
    """
&'s avatar
metrics  
& committed
133
134
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
135
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
136
137
138
    return sacrebleu.corpus_ter(preds, refs).score


&'s avatar
& committed
139
140
141
142
def is_non_str_iterable(obj):
    return isinstance(obj, Iterable) and not isinstance(obj, str)


&'s avatar
& committed
143
def _sacreformat(refs, preds):
&'s avatar
metrics  
& committed
144
    """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
&'s avatar
& committed
145
    # Sacrebleu expects (List[str], List[List[str])
&'s avatar
metrics  
& committed
146
147
    #   e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])

&'s avatar
& committed
148
149
150
151
152
153
    # Note [ref1_stream] is the first reference for each pred.
    # So lists are size N and (M, N) for N preds and M possible refs for each pred
    # This is a different order of dimensions that I would expect

    # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
    # Must become List[List[str]] with the inner list corresponding to preds
&'s avatar
& committed
154
    if not is_non_str_iterable(refs):
&'s avatar
metrics  
& committed
155
        refs = list(refs)
&'s avatar
& committed
156
    if not is_non_str_iterable(refs[0]):
&'s avatar
metrics  
& committed
157
        refs = [[ref] for ref in refs]
&'s avatar
& committed
158
159
    refs = list(zip(*refs))
    # Note the number of refs in each ref list much match the number of preds
&'s avatar
metrics  
& committed
160

&'s avatar
& committed
161
    # We expect preds to be List[str] or List[List[str]]. Must become List[str]
&'s avatar
& committed
162
    if not is_non_str_iterable(preds):
&'s avatar
metrics  
& committed
163
        preds = list(preds)
&'s avatar
& committed
164
    if is_non_str_iterable(preds[0]):
&'s avatar
& committed
165
166
        assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
        preds = [pred[0] for pred in preds]
&'s avatar
metrics  
& committed
167
168

    return refs, preds
Leo Gao's avatar
Leo Gao committed
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

## stderr stuff


def bootstrap_stddev(f, xs, iters=10000):
    rnd = random.Random()
    rnd.seed(42)
    res = []
    from tqdm import trange
    print("bootstrapping for stddev:", f.__name__)
    for i in trange(iters):
        # sample w replacement
        bootstrap = rnd.choices(xs, k=len(xs))
        res.append(stddev(bootstrap))

    return mean(res)


def stderr_for_metric(metric):
    bootstrappable = [
        median,
        matthews_corrcoef,
        f1_score,
        perplexity,
        bleu,
        chrf,
        ter,
    ]

    if metric in bootstrappable:
        return lambda x: bootstrap_stddev(metric, x) / math.sqrt(len(x))

    stderr = {
        mean: mean_stderr,
        acc_all: acc_all_stderr
        
    }

    return stderr.get(metric, None)