metrics.py 7.82 KB
Newer Older
&'s avatar
& committed
1
import math
2
from collections.abc import Iterable
&'s avatar
& committed
3
4
5

import numpy as np
import sacrebleu
Jonathan Tow's avatar
Jonathan Tow committed
6
import sklearn.metrics
Leo Gao's avatar
Leo Gao committed
7
import random
&'s avatar
& committed
8
9
10


def mean(arr):
fromSun2Moon's avatar
fromSun2Moon committed
11
    print(len(arr))
&'s avatar
& committed
12
13
14
    return sum(arr) / len(arr)


Leo Gao's avatar
Leo Gao committed
15
def pop_stddev(arr):
Leo Gao's avatar
Leo Gao committed
16
17
18
19
    mu = mean(arr)
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))


Leo Gao's avatar
Leo Gao committed
20
21
22
23
24
def sample_stddev(arr):
    mu = mean(arr)
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))


Leo Gao's avatar
Leo Gao committed
25
def mean_stderr(arr):
Leo Gao's avatar
Leo Gao committed
26
    return sample_stddev(arr) / math.sqrt(len(arr))
Leo Gao's avatar
Leo Gao committed
27
28


&'s avatar
& committed
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def median(arr):
    return arr[len(arr) // 2]


def matthews_corrcoef(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    return sklearn.metrics.matthews_corrcoef(golds, preds)


def f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    fscore = sklearn.metrics.f1_score(golds, preds)
    return np.max(fscore)

47
48
49
50
51
52
def macro_f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    fscore = sklearn.metrics.f1_score(golds, preds, average='macro')
    return fscore
&'s avatar
& committed
53
54
55
56
57
58
59
60

def acc_all(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
61
        paragraph_id = doc["idx"]["paragraph"]
&'s avatar
& committed
62
        question_id = doc["idx"]["question"]
63
64
        if (paragraph_id, question_id) not in question_scoring_dict:
            question_scoring_dict[(paragraph_id, question_id)] = []
&'s avatar
& committed
65
66
67

        gold_label = doc["label"] == 1

68
        question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred)
&'s avatar
& committed
69
70
71
    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
    return acc

72

Leo Gao's avatar
Leo Gao committed
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def acc_all_stderr(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
    return acc

&'s avatar
& committed
90
91
92
93
94
95
96
97
98
99
100
101
102
103

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


def perplexity(items):
    return math.exp(-mean(items))


Leo Gao's avatar
Leo Gao committed
104
105
106
107
def weighted_mean(items):
    a, b = zip(*items)
    return sum(a) / sum(b)

108

Leo Gao's avatar
Leo Gao committed
109
110
111
def weighted_perplexity(items):
    return math.exp(-weighted_mean(items))

Fabrizio Milo's avatar
Fabrizio Milo committed
112

113
114
115
def bits_per_byte(items):
    return -weighted_mean(items) / math.log(2)

Leo Gao's avatar
Leo Gao committed
116

&'s avatar
& committed
117
118
119
120
121
122
123
124
125
126
127
def bleu(items):
    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
    for evaluating a generated sentence to a reference sentence. It counts matching
    n-grams in the candidate translation to n-grams in the reference text, where
    1-gram or unigram would be each token and a bigram comparison would be each
    word pair. The comparison is made regardless of word order
    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
    Paper: https://www.aclweb.org/anthology/P02-1040/

    Higher is better
    """
&'s avatar
metrics  
& committed
128
129
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
130
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
131
132
    return sacrebleu.corpus_bleu(preds, refs).score

&'s avatar
& committed
133
134
135
136
137
138
139
140
141

def chrf(items):
    """chrF++ is a tool for automatic evaluation of machine translation output
    based on character n-gram precision and recall enhanced with word n-grams.
    Source: https://github.com/m-popovic/chrF
    Paper: https://www.aclweb.org/anthology/W15-3049.pdf

    Higher is better  # TODO I think
    """
&'s avatar
metrics  
& committed
142
143
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
144
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
145
146
    return sacrebleu.corpus_chrf(preds, refs).score

&'s avatar
& committed
147
148
149
150
151
152
153
154
155
156

def ter(items):
    """Translation Error Rate is an error metric for machine translation that
    measures the number of edits required to change a system output into one
    of the references
    Source: http://www.cs.umd.edu/~snover/tercom/
    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf

    Lower is better
    """
&'s avatar
metrics  
& committed
157
158
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
159
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
160
161
162
    return sacrebleu.corpus_ter(preds, refs).score


&'s avatar
& committed
163
164
165
166
def is_non_str_iterable(obj):
    return isinstance(obj, Iterable) and not isinstance(obj, str)


&'s avatar
& committed
167
def _sacreformat(refs, preds):
&'s avatar
metrics  
& committed
168
    """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
&'s avatar
& committed
169
    # Sacrebleu expects (List[str], List[List[str])
&'s avatar
metrics  
& committed
170
171
    #   e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])

&'s avatar
& committed
172
173
174
175
176
177
    # Note [ref1_stream] is the first reference for each pred.
    # So lists are size N and (M, N) for N preds and M possible refs for each pred
    # This is a different order of dimensions that I would expect

    # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
    # Must become List[List[str]] with the inner list corresponding to preds
&'s avatar
& committed
178
    if not is_non_str_iterable(refs):
&'s avatar
metrics  
& committed
179
        refs = list(refs)
&'s avatar
& committed
180
    if not is_non_str_iterable(refs[0]):
&'s avatar
metrics  
& committed
181
        refs = [[ref] for ref in refs]
&'s avatar
& committed
182
183
    refs = list(zip(*refs))
    # Note the number of refs in each ref list much match the number of preds
&'s avatar
metrics  
& committed
184

&'s avatar
& committed
185
    # We expect preds to be List[str] or List[List[str]]. Must become List[str]
&'s avatar
& committed
186
    if not is_non_str_iterable(preds):
&'s avatar
metrics  
& committed
187
        preds = list(preds)
&'s avatar
& committed
188
    if is_non_str_iterable(preds[0]):
&'s avatar
& committed
189
190
        assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
        preds = [pred[0] for pred in preds]
&'s avatar
metrics  
& committed
191
192

    return refs, preds
Leo Gao's avatar
Leo Gao committed
193

Fabrizio Milo's avatar
Fabrizio Milo committed
194

195
# stderr stuff
Leo Gao's avatar
Leo Gao committed
196
197
198
199
class _bootstrap_internal:
    def __init__(self, f, n):
        self.f = f
        self.n = n
200

Leo Gao's avatar
Leo Gao committed
201
202
203
204
205
206
207
208
209
    def __call__(self, v):
        i, xs = v
        rnd = random.Random()
        rnd.seed(i)
        res = []
        for _ in range(self.n):
            res.append(self.f(rnd.choices(xs, k=len(xs))))
        return res

Leo Gao's avatar
Leo Gao committed
210

211
def bootstrap_stderr(f, xs, iters):
Leo Gao's avatar
Leo Gao committed
212
    import multiprocessing as mp
Fabrizio Milo's avatar
Fabrizio Milo committed
213

Leo Gao's avatar
Leo Gao committed
214
    pool = mp.Pool(mp.cpu_count())
Leo Gao's avatar
Leo Gao committed
215
    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
Fabrizio Milo's avatar
Fabrizio Milo committed
216
    # equivalent to stderr calculated without Bessel's correction in the stddev.
Leo Gao's avatar
Leo Gao committed
217
218
219
220
    # Unfortunately, I haven't been able to figure out what the right correction is
    # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
    # Thankfully, shouldn't matter because our samples are pretty big usually anyways
Leo Gao's avatar
Leo Gao committed
221
    res = []
222
    chunk_size = min(1000, iters)
Leo Gao's avatar
Leo Gao committed
223
    from tqdm import tqdm
Fabrizio Milo's avatar
Fabrizio Milo committed
224

Leo Gao's avatar
Leo Gao committed
225
    print("bootstrapping for stddev:", f.__name__)
Fabrizio Milo's avatar
Fabrizio Milo committed
226
227
    for bootstrap in tqdm(
        pool.imap(
228
            _bootstrap_internal(f, chunk_size),
Fabrizio Milo's avatar
Fabrizio Milo committed
229
230
231
232
            [(i, xs) for i in range(iters // chunk_size)],
        ),
        total=iters // chunk_size,
    ):
Leo Gao's avatar
Leo Gao committed
233
        # sample w replacement
Leo Gao's avatar
Leo Gao committed
234
        res.extend(bootstrap)
Leo Gao's avatar
Leo Gao committed
235

Leo Gao's avatar
Leo Gao committed
236
    pool.close()
Leo Gao's avatar
Leo Gao committed
237
    return sample_stddev(res)
Leo Gao's avatar
Leo Gao committed
238
239


240
def stderr_for_metric(metric, bootstrap_iters):
Leo Gao's avatar
Leo Gao committed
241
242
243
244
245
246
247
248
    bootstrappable = [
        median,
        matthews_corrcoef,
        f1_score,
        perplexity,
        bleu,
        chrf,
        ter,
kabbi159's avatar
kabbi159 committed
249
        macro_f1_score,
Leo Gao's avatar
Leo Gao committed
250
251
252
    ]

    if metric in bootstrappable:
253
        return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters)
Leo Gao's avatar
Leo Gao committed
254

Fabrizio Milo's avatar
Fabrizio Milo committed
255
    stderr = {mean: mean_stderr, acc_all: acc_all_stderr}
Leo Gao's avatar
Leo Gao committed
256

Leo Gao's avatar
Leo Gao committed
257
    return stderr.get(metric, None)
Jonathan Tow's avatar
Jonathan Tow committed
258
259
260
261


def yesno(x):
    if x:
Fabrizio Milo's avatar
Fabrizio Milo committed
262
        return "yes"
Jonathan Tow's avatar
Jonathan Tow committed
263
    else:
Fabrizio Milo's avatar
Fabrizio Milo committed
264
        return "no"