metrics.py 9.95 KB
Newer Older
&'s avatar
& committed
1
import math
2
from collections.abc import Iterable
&'s avatar
& committed
3
4
5

import numpy as np
import sacrebleu
Jonathan Tow's avatar
Jonathan Tow committed
6
import sklearn.metrics
Leo Gao's avatar
Leo Gao committed
7
import random
&'s avatar
& committed
8

haileyschoelkopf's avatar
haileyschoelkopf committed
9
import evaluate
&'s avatar
& committed
10

haileyschoelkopf's avatar
haileyschoelkopf committed
11
12

AGGREGATION_REGISTRY = {}
lintangsutawika's avatar
lintangsutawika committed
13

14
15
METRIC_REGISTRY = {
    "acc": None,
haileyschoelkopf's avatar
haileyschoelkopf committed
16
    "acc_norm": None,
17
    "acc_mutual_info": None,
haileyschoelkopf's avatar
haileyschoelkopf committed
18
19
    "word_perplexity": None,
    "byte_perplexity": None,
20
}
haileyschoelkopf's avatar
haileyschoelkopf committed
21

lintangsutawika's avatar
lintangsutawika committed
22
23
24
25
26
27
28
29
30
31
32
33
34
35
HIGHER_IS_BETTER_REGISTRY = {
    "matthews_corrcoef": True,
    "f1_score": True,
    "perplexity": False,
    "bleu": True,
    "chrf": True,
    "ter": False,
    "acc": True,
    "acc_norm": True,
    "acc_mutual_info": True,
    "word_perplexity": False,
    "byte_perplexity": False,
    "bits_per_byte": False,
}
haileyschoelkopf's avatar
haileyschoelkopf committed
36

lintangsutawika's avatar
lintangsutawika committed
37

haileyschoelkopf's avatar
haileyschoelkopf committed
38
39
40
41
42
43
44
45
46
def register_metric(name):
    # TODO: do we want to enforce a certain interface to registered metrics?
    def decorate(fn):
        assert (
            name not in METRIC_REGISTRY
        ), f"metric named '{name}' conflicts with existing registered metric!"

        METRIC_REGISTRY[name] = fn
        return fn
lintangsutawika's avatar
lintangsutawika committed
47

haileyschoelkopf's avatar
haileyschoelkopf committed
48
49
50
51
52
53
54
55
56
    return decorate


def get_metric(name):

    try:
        return METRIC_REGISTRY[name]
    except KeyError:
        # TODO: change this print to logging?
lintangsutawika's avatar
lintangsutawika committed
57
58
59
60
        print(
            f"Could not find registered metric '{name}' in lm-eval, \
searching in HF Evaluate library..."
        )
haileyschoelkopf's avatar
haileyschoelkopf committed
61
62
63
        try:
            metric_object = evaluate.load(name)
            return metric_object.compute
lintangsutawika's avatar
lintangsutawika committed
64
        except Exception:
haileyschoelkopf's avatar
haileyschoelkopf committed
65
66
67
68
69
70
71
            raise Warning(
                "{} not found in the evaluate library!".format(name),
                "Please check https://huggingface.co/evaluate-metric",
            )


def register_aggregation(name):
72
    # TODO: should we enforce a specific interface to aggregation metrics?
haileyschoelkopf's avatar
haileyschoelkopf committed
73
74
75
76
77
78
79
    def decorate(fn):
        assert (
            name not in AGGREGATION_REGISTRY
        ), f"aggregation named '{name}' conflicts with existing registered aggregation!"

        AGGREGATION_REGISTRY[name] = fn
        return fn
lintangsutawika's avatar
lintangsutawika committed
80

haileyschoelkopf's avatar
haileyschoelkopf committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    return decorate


def get_aggregation(name):

    try:
        return AGGREGATION_REGISTRY[name]
    except KeyError:
        raise Warning(
            "{} not a registered aggregation metric!".format(name),
        )


@register_aggregation("mean")
&'s avatar
& committed
95
96
97
98
def mean(arr):
    return sum(arr) / len(arr)


Leo Gao's avatar
Leo Gao committed
99
def pop_stddev(arr):
Leo Gao's avatar
Leo Gao committed
100
101
102
103
    mu = mean(arr)
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))


Leo Gao's avatar
Leo Gao committed
104
105
106
107
108
def sample_stddev(arr):
    mu = mean(arr)
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))


Leo Gao's avatar
Leo Gao committed
109
def mean_stderr(arr):
Leo Gao's avatar
Leo Gao committed
110
    return sample_stddev(arr) / math.sqrt(len(arr))
Leo Gao's avatar
Leo Gao committed
111
112


haileyschoelkopf's avatar
haileyschoelkopf committed
113
@register_aggregation("median")
&'s avatar
& committed
114
115
116
117
def median(arr):
    return arr[len(arr) // 2]


haileyschoelkopf's avatar
haileyschoelkopf committed
118
@register_metric("matthews_corrcoef")
&'s avatar
& committed
119
120
121
122
123
124
125
def matthews_corrcoef(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    return sklearn.metrics.matthews_corrcoef(golds, preds)


haileyschoelkopf's avatar
haileyschoelkopf committed
126
@register_metric("f1_score")
&'s avatar
& committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
def f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    fscore = sklearn.metrics.f1_score(golds, preds)

    return np.max(fscore)


def acc_all(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
143
        paragraph_id = doc["idx"]["paragraph"]
&'s avatar
& committed
144
        question_id = doc["idx"]["question"]
145
146
        if (paragraph_id, question_id) not in question_scoring_dict:
            question_scoring_dict[(paragraph_id, question_id)] = []
&'s avatar
& committed
147
148
149

        gold_label = doc["label"] == 1

150
        question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred)
&'s avatar
& committed
151
152
153
    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
    return acc

154

Leo Gao's avatar
Leo Gao committed
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
def acc_all_stderr(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
    return acc

&'s avatar
& committed
172
173
174
175
176
177
178
179
180
181

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


haileyschoelkopf's avatar
haileyschoelkopf committed
182
@register_metric("perplexity")
183
@register_aggregation("perplexity")
&'s avatar
& committed
184
185
186
187
def perplexity(items):
    return math.exp(-mean(items))


Leo Gao's avatar
Leo Gao committed
188
189
190
191
def weighted_mean(items):
    a, b = zip(*items)
    return sum(a) / sum(b)

192

haileyschoelkopf's avatar
haileyschoelkopf committed
193
@register_metric("weighted_perplexity")
haileyschoelkopf's avatar
haileyschoelkopf committed
194
@register_aggregation("weighted_perplexity")
Leo Gao's avatar
Leo Gao committed
195
196
197
def weighted_perplexity(items):
    return math.exp(-weighted_mean(items))

Fabrizio Milo's avatar
Fabrizio Milo committed
198

haileyschoelkopf's avatar
haileyschoelkopf committed
199
200
@register_metric("bits_per_byte")
@register_aggregation("bits_per_byte")
201
202
203
def bits_per_byte(items):
    return -weighted_mean(items) / math.log(2)

Leo Gao's avatar
Leo Gao committed
204

haileyschoelkopf's avatar
haileyschoelkopf committed
205
@register_metric("bleu")
&'s avatar
& committed
206
207
208
209
210
211
212
213
214
215
216
def bleu(items):
    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
    for evaluating a generated sentence to a reference sentence. It counts matching
    n-grams in the candidate translation to n-grams in the reference text, where
    1-gram or unigram would be each token and a bigram comparison would be each
    word pair. The comparison is made regardless of word order
    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
    Paper: https://www.aclweb.org/anthology/P02-1040/

    Higher is better
    """
&'s avatar
metrics  
& committed
217
218
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
219
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
220
221
    return sacrebleu.corpus_bleu(preds, refs).score

&'s avatar
& committed
222

haileyschoelkopf's avatar
haileyschoelkopf committed
223
@register_metric("chrf")
&'s avatar
& committed
224
225
226
227
228
229
230
231
def chrf(items):
    """chrF++ is a tool for automatic evaluation of machine translation output
    based on character n-gram precision and recall enhanced with word n-grams.
    Source: https://github.com/m-popovic/chrF
    Paper: https://www.aclweb.org/anthology/W15-3049.pdf

    Higher is better  # TODO I think
    """
&'s avatar
metrics  
& committed
232
233
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
234
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
235
236
    return sacrebleu.corpus_chrf(preds, refs).score

&'s avatar
& committed
237

haileyschoelkopf's avatar
haileyschoelkopf committed
238
@register_metric("ter")
&'s avatar
& committed
239
240
241
242
243
244
245
246
247
def ter(items):
    """Translation Error Rate is an error metric for machine translation that
    measures the number of edits required to change a system output into one
    of the references
    Source: http://www.cs.umd.edu/~snover/tercom/
    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf

    Lower is better
    """
&'s avatar
metrics  
& committed
248
249
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
&'s avatar
& committed
250
    refs, preds = _sacreformat(refs, preds)
&'s avatar
metrics  
& committed
251
252
253
    return sacrebleu.corpus_ter(preds, refs).score


&'s avatar
& committed
254
255
256
257
def is_non_str_iterable(obj):
    return isinstance(obj, Iterable) and not isinstance(obj, str)


&'s avatar
& committed
258
def _sacreformat(refs, preds):
&'s avatar
metrics  
& committed
259
    """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
&'s avatar
& committed
260
    # Sacrebleu expects (List[str], List[List[str])
&'s avatar
metrics  
& committed
261
262
    #   e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])

&'s avatar
& committed
263
264
265
266
267
268
    # Note [ref1_stream] is the first reference for each pred.
    # So lists are size N and (M, N) for N preds and M possible refs for each pred
    # This is a different order of dimensions that I would expect

    # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
    # Must become List[List[str]] with the inner list corresponding to preds
&'s avatar
& committed
269
    if not is_non_str_iterable(refs):
&'s avatar
metrics  
& committed
270
        refs = list(refs)
&'s avatar
& committed
271
    if not is_non_str_iterable(refs[0]):
&'s avatar
metrics  
& committed
272
        refs = [[ref] for ref in refs]
&'s avatar
& committed
273
274
    refs = list(zip(*refs))
    # Note the number of refs in each ref list much match the number of preds
&'s avatar
metrics  
& committed
275

&'s avatar
& committed
276
    # We expect preds to be List[str] or List[List[str]]. Must become List[str]
&'s avatar
& committed
277
    if not is_non_str_iterable(preds):
&'s avatar
metrics  
& committed
278
        preds = list(preds)
&'s avatar
& committed
279
    if is_non_str_iterable(preds[0]):
&'s avatar
& committed
280
281
        assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
        preds = [pred[0] for pred in preds]
&'s avatar
metrics  
& committed
282
283

    return refs, preds
Leo Gao's avatar
Leo Gao committed
284

Fabrizio Milo's avatar
Fabrizio Milo committed
285

286
# stderr stuff
Leo Gao's avatar
Leo Gao committed
287

Fabrizio Milo's avatar
Fabrizio Milo committed
288

Leo Gao's avatar
Leo Gao committed
289
290
291
292
class _bootstrap_internal:
    def __init__(self, f, n):
        self.f = f
        self.n = n
293

Leo Gao's avatar
Leo Gao committed
294
295
296
297
298
299
300
301
302
    def __call__(self, v):
        i, xs = v
        rnd = random.Random()
        rnd.seed(i)
        res = []
        for _ in range(self.n):
            res.append(self.f(rnd.choices(xs, k=len(xs))))
        return res

Leo Gao's avatar
Leo Gao committed
303

304
def bootstrap_stderr(f, xs, iters):
Leo Gao's avatar
Leo Gao committed
305
    import multiprocessing as mp
Fabrizio Milo's avatar
Fabrizio Milo committed
306

Leo Gao's avatar
Leo Gao committed
307
    pool = mp.Pool(mp.cpu_count())
Leo Gao's avatar
Leo Gao committed
308
    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
Fabrizio Milo's avatar
Fabrizio Milo committed
309
    # equivalent to stderr calculated without Bessel's correction in the stddev.
Leo Gao's avatar
Leo Gao committed
310
311
312
313
    # Unfortunately, I haven't been able to figure out what the right correction is
    # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
    # Thankfully, shouldn't matter because our samples are pretty big usually anyways
Leo Gao's avatar
Leo Gao committed
314
    res = []
315
    chunk_size = min(1000, iters)
Leo Gao's avatar
Leo Gao committed
316
    from tqdm import tqdm
Fabrizio Milo's avatar
Fabrizio Milo committed
317

Leo Gao's avatar
Leo Gao committed
318
    print("bootstrapping for stddev:", f.__name__)
Fabrizio Milo's avatar
Fabrizio Milo committed
319
320
    for bootstrap in tqdm(
        pool.imap(
321
            _bootstrap_internal(f, chunk_size),
Fabrizio Milo's avatar
Fabrizio Milo committed
322
323
324
325
            [(i, xs) for i in range(iters // chunk_size)],
        ),
        total=iters // chunk_size,
    ):
Leo Gao's avatar
Leo Gao committed
326
        # sample w replacement
Leo Gao's avatar
Leo Gao committed
327
        res.extend(bootstrap)
Leo Gao's avatar
Leo Gao committed
328

Leo Gao's avatar
Leo Gao committed
329
    pool.close()
Leo Gao's avatar
Leo Gao committed
330
    return sample_stddev(res)
Leo Gao's avatar
Leo Gao committed
331
332


333
def stderr_for_metric(metric, bootstrap_iters):
Leo Gao's avatar
Leo Gao committed
334
335
336
337
338
339
340
341
342
343
344
    bootstrappable = [
        median,
        matthews_corrcoef,
        f1_score,
        perplexity,
        bleu,
        chrf,
        ter,
    ]

    if metric in bootstrappable:
345
        return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters)
Leo Gao's avatar
Leo Gao committed
346

Fabrizio Milo's avatar
Fabrizio Milo committed
347
    stderr = {mean: mean_stderr, acc_all: acc_all_stderr}
Leo Gao's avatar
Leo Gao committed
348

Leo Gao's avatar
Leo Gao committed
349
    return stderr.get(metric, None)
Jonathan Tow's avatar
Jonathan Tow committed
350
351
352
353


def yesno(x):
    if x:
Fabrizio Milo's avatar
Fabrizio Milo committed
354
        return "yes"
Jonathan Tow's avatar
Jonathan Tow committed
355
    else:
Fabrizio Milo's avatar
Fabrizio Milo committed
356
        return "no"