metrics.py 16.5 KB
Newer Older
1
import logging
&'s avatar
& committed
2
import math
3
import random
4
5
import re
import string
6
from collections.abc import Iterable
7
from typing import List
8
9
10

import numpy as np
import sacrebleu
&'s avatar
& committed
11

12
from lm_eval.api.registry import register_aggregation, register_metric
13

lintangsutawika's avatar
lintangsutawika committed
14

Lintang Sutawika's avatar
Lintang Sutawika committed
15
eval_logger = logging.getLogger(__name__)
16

17

18
# Register Aggregations First
Baber Abbasi's avatar
Baber Abbasi committed
19
20
21
22
23
@register_aggregation("bypass")
def bypass_agg(arr):
    return 999


24
25
26
27
28
29
30
@register_aggregation("nanmean")
def nanmean(arr):
    if len(arr) == 0 or all(np.isnan(arr)):
        return np.nan
    return np.nanmean(arr)


31
32
33
34
35
36
37
38
39
40
@register_aggregation("mean")
def mean(arr):
    return sum(arr) / len(arr)


@register_aggregation("median")
def median(arr):
    return arr[len(arr) // 2]


41
# Certain metrics must be calculated across all documents in a benchmark.
haileyschoelkopf's avatar
haileyschoelkopf committed
42
# We use them as aggregation metrics, paired with no-op passthrough metric fns.
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
@register_aggregation("perplexity")
def perplexity(items):
    return math.exp(-mean(items))


@register_aggregation("weighted_perplexity")
def weighted_perplexity(items):
    return math.exp(-weighted_mean(items))


@register_aggregation("bits_per_byte")
def bits_per_byte(items):
    return -weighted_mean(items) / math.log(2)


haileyschoelkopf's avatar
haileyschoelkopf committed
58
59
@register_aggregation("f1")
def f1_score(items):
60
61
    from sklearn.metrics import f1_score

haileyschoelkopf's avatar
haileyschoelkopf committed
62
63
64
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
65
    fscore = f1_score(golds, preds)
haileyschoelkopf's avatar
haileyschoelkopf committed
66
67
68
69
70
71

    return np.max(fscore)


@register_aggregation("matthews_corrcoef")
def matthews_corrcoef(items):
72
73
    from sklearn.metrics import matthews_corrcoef

haileyschoelkopf's avatar
haileyschoelkopf committed
74
75
76
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
77
    return matthews_corrcoef(golds, preds)
haileyschoelkopf's avatar
haileyschoelkopf committed
78
79


80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
@register_aggregation("bleu")
def bleu(items):
    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
    for evaluating a generated sentence to a reference sentence. It counts matching
    n-grams in the candidate translation to n-grams in the reference text, where
    1-gram or unigram would be each token and a bigram comparison would be each
    word pair. The comparison is made regardless of word order
    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
    Paper: https://www.aclweb.org/anthology/P02-1040/

    Higher is better
    """
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
    refs, preds = _sacreformat(refs, preds)
    return sacrebleu.corpus_bleu(preds, refs).score


98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
@register_aggregation("chrf")
def chrf(items):
    """chrF++ is a tool for automatic evaluation of machine translation output
    based on character n-gram precision and recall enhanced with word n-grams.
    Source: https://github.com/m-popovic/chrF
    Paper: https://www.aclweb.org/anthology/W15-3049.pdf

    Higher is better  # TODO I think
    """
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
    refs, preds = _sacreformat(refs, preds)
    return sacrebleu.corpus_chrf(preds, refs).score


@register_aggregation("ter")
def ter(items):
    """Translation Error Rate is an error metric for machine translation that
    measures the number of edits required to change a system output into one
    of the references
    Source: http://www.cs.umd.edu/~snover/tercom/
    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf

    Lower is better
    """
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
    refs, preds = _sacreformat(refs, preds)
    return sacrebleu.corpus_ter(preds, refs).score


Lintang Sutawika's avatar
Lintang Sutawika committed
129
130
131
@register_aggregation("brier_score")
def brier_score(items):  # This is a passthrough function
    gold, predictions = list(zip(*items))
Lintang Sutawika's avatar
Lintang Sutawika committed
132
133
    bs, num_class = np.array(predictions).shape

Lintang Sutawika's avatar
Lintang Sutawika committed
134
    gold = list(gold)
Lintang Sutawika's avatar
Lintang Sutawika committed
135
    gold_one_hot = np.eye(num_class)[gold]
Lintang Sutawika's avatar
Lintang Sutawika committed
136
137
138
139
140
141
142
143
144
145
146
147
148
    return np.mean(np.sum((predictions - gold_one_hot) ** 2, axis=1))


@register_metric(
    metric="brier_score",
    higher_is_better=False,
    output_type=["multiple_choice"],
    aggregation="brier_score",
)
def brier_score_fn(items):  # This is a passthrough function
    return items


149
150
151
152
153
154
155
156
157
158
@register_metric(
    metric="acc",
    higher_is_better=True,
    output_type=["loglikelihood", "multiple_choice"],
    aggregation="mean",
)
def acc_fn(items):  # This is a passthrough function
    return items


159
160
161
162
163
164
165
166
167
168
@register_metric(
    metric="acc_norm",
    higher_is_better=True,
    output_type=["loglikelihood", "multiple_choice"],
    aggregation="mean",
)
def acc_norm_fn(items):  # This is a passthrough function
    return items


169
170
171
172
173
174
175
176
177
178
@register_metric(
    metric="acc_mutual_info",
    higher_is_better=True,
    output_type="multiple_choice",
    aggregation="mean",
)
def acc_mutual_info_fn(items):  # This is a passthrough function
    return items


179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
### the code used in the `exact_match_hf_evaluate` function is ported from
### https://github.com/huggingface/evaluate/blob/main/metrics/exact_match/exact_match.py
### which is under the apache license.

# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0


# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def exact_match_hf_evaluate(
    predictions,
    references,
    regexes_to_ignore=None,
    ignore_case=False,
    ignore_punctuation=False,
    ignore_numbers=False,
):
    if regexes_to_ignore is not None:
        for s in regexes_to_ignore:
            predictions = np.array([re.sub(s, "", x) for x in predictions])
            references = np.array([re.sub(s, "", x) for x in references])
    else:
        predictions = np.asarray(predictions)
        references = np.asarray(references)

    if ignore_case:
        predictions = np.char.lower(predictions)
        references = np.char.lower(references)

    if ignore_punctuation:
        repl_table = string.punctuation.maketrans("", "", string.punctuation)
        predictions = np.char.translate(predictions, table=repl_table)
        references = np.char.translate(references, table=repl_table)

    if ignore_numbers:
        repl_table = string.digits.maketrans("", "", string.digits)
        predictions = np.char.translate(predictions, table=repl_table)
        references = np.char.translate(references, table=repl_table)

    score_list = predictions == references

    return {"exact_match": np.mean(score_list)}


###
233
234


235
236
237
238
239
240
@register_metric(
    metric="exact_match",
    higher_is_better=True,
    output_type="generate_until",
    aggregation="mean",
)
241
def exact_match_fn(**kwargs):
242
    return exact_match_hf_evaluate(**kwargs)
243
244


245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
@register_metric(
    metric="perplexity",
    higher_is_better=False,
    output_type="loglikelihood",
    aggregation="perplexity",
)
def perplexity_fn(items):  # This is a passthrough function
    return items


@register_metric(
    metric="word_perplexity",
    higher_is_better=False,
    output_type="loglikelihood_rolling",
    aggregation="weighted_perplexity",
)
def word_perplexity_fn(items):  # This is a passthrough function
    return items


@register_metric(
    metric="byte_perplexity",
    higher_is_better=False,
    output_type="loglikelihood_rolling",
    aggregation="weighted_perplexity",
)
def byte_perplexity_fn(items):  # This is a passthrough function
    return items


@register_metric(
    metric="bits_per_byte",
    higher_is_better=False,
    output_type="loglikelihood_rolling",
    aggregation="bits_per_byte",
)
def bits_per_byte_fn(items):  # This is a passthrough function
    return items

&'s avatar
& committed
284

Leo Gao's avatar
Leo Gao committed
285
def pop_stddev(arr):
286
    mu = mean(arr)
Leo Gao's avatar
Leo Gao committed
287
288
289
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))


Leo Gao's avatar
Leo Gao committed
290
def sample_stddev(arr):
291
    mu = mean(arr)
Leo Gao's avatar
Leo Gao committed
292
293
294
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))


Leo Gao's avatar
Leo Gao committed
295
def mean_stderr(arr):
Leo Gao's avatar
Leo Gao committed
296
    return sample_stddev(arr) / math.sqrt(len(arr))
Leo Gao's avatar
Leo Gao committed
297
298


Baber Abbasi's avatar
Baber Abbasi committed
299
300
301
302
303
304
305
306
307
308
@register_metric(
    metric="bypass",
    higher_is_better=True,
    output_type=["loglikelihood", "multiple_choice", "generate_until"],
    aggregation="bypass",
)
def bypass(items):
    return None


haileyschoelkopf's avatar
haileyschoelkopf committed
309
310
311
312
313
314
315
316
@register_metric(
    metric="mcc",
    higher_is_better=True,
    output_type="multiple_choice",
    aggregation="matthews_corrcoef",
)
def mcc_fn(items):  # This is a passthrough function
    return items
317
318
319


@register_metric(
320
    metric="f1",
321
322
    higher_is_better=True,
    output_type="multiple_choice",
haileyschoelkopf's avatar
haileyschoelkopf committed
323
    aggregation="f1",
324
)
325
def f1_fn(items):  # This is a passthrough function
haileyschoelkopf's avatar
haileyschoelkopf committed
326
    return items
327
328


329
330
331
@register_metric(
    metric="bleu",
    higher_is_better=True,
332
    output_type="generate_until",
333
334
335
336
337
338
    aggregation="bleu",
)
def bleu_fn(items):  # This is a passthrough function
    return items


339
340
341
@register_metric(
    metric="chrf",
    higher_is_better=True,
342
    output_type="generate_until",
343
344
345
346
347
348
349
350
351
    aggregation="chrf",
)
def chrf_fn(items):  # This is a passthrough function
    return items


@register_metric(
    metric="ter",
    higher_is_better=True,
352
    output_type="generate_until",
353
354
355
356
357
358
    aggregation="ter",
)
def ter_fn(items):  # This is a passthrough function
    return items


359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
@register_metric(
    metric="acc_all",
    higher_is_better=True,
    output_type="loglikelihood",
    aggregation="mean",
)
def acc_all(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        paragraph_id = doc["idx"]["paragraph"]
        question_id = doc["idx"]["question"]
        if (paragraph_id, question_id) not in question_scoring_dict:
            question_scoring_dict[(paragraph_id, question_id)] = []

        gold_label = doc["label"] == 1

        question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred)
    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
    return acc


Leo Gao's avatar
Leo Gao committed
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
def acc_all_stderr(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
    return acc

&'s avatar
& committed
401
402
403
404
405
406
407
408
409
410

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
def weighted_mean(items):
    a, b = zip(*items)
    return sum(a) / sum(b)


def is_non_str_iterable(obj):
    return isinstance(obj, Iterable) and not isinstance(obj, str)


def _sacreformat(refs, preds):
    """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
    # Sacrebleu expects (List[str], List[List[str])
    #   e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])

    # Note [ref1_stream] is the first reference for each pred.
    # So lists are size N and (M, N) for N preds and M possible refs for each pred
    # This is a different order of dimensions that I would expect

    # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
    # Must become List[List[str]] with the inner list corresponding to preds
    if not is_non_str_iterable(refs):
        refs = list(refs)
    if not is_non_str_iterable(refs[0]):
        refs = [[ref] for ref in refs]
    refs = list(zip(*refs))
    # Note the number of refs in each ref list much match the number of preds

    # We expect preds to be List[str] or List[List[str]]. Must become List[str]
    if not is_non_str_iterable(preds):
        preds = list(preds)
    if is_non_str_iterable(preds[0]):
        assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
        preds = [pred[0] for pred in preds]

    return refs, preds


# stderr stuff


Leo Gao's avatar
Leo Gao committed
451
class _bootstrap_internal:
Ethan Smith's avatar
Ethan Smith committed
452
    def __init__(self, f, n) -> None:
Leo Gao's avatar
Leo Gao committed
453
454
        self.f = f
        self.n = n
455

Leo Gao's avatar
Leo Gao committed
456
457
458
459
460
461
462
463
464
    def __call__(self, v):
        i, xs = v
        rnd = random.Random()
        rnd.seed(i)
        res = []
        for _ in range(self.n):
            res.append(self.f(rnd.choices(xs, k=len(xs))))
        return res

Leo Gao's avatar
Leo Gao committed
465

466
def bootstrap_stderr(f, xs, iters):
Leo Gao's avatar
Leo Gao committed
467
    import multiprocessing as mp
Fabrizio Milo's avatar
Fabrizio Milo committed
468

Leo Gao's avatar
Leo Gao committed
469
    pool = mp.Pool(mp.cpu_count())
Leo Gao's avatar
Leo Gao committed
470
    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
Fabrizio Milo's avatar
Fabrizio Milo committed
471
    # equivalent to stderr calculated without Bessel's correction in the stddev.
Leo Gao's avatar
Leo Gao committed
472
473
474
475
    # Unfortunately, I haven't been able to figure out what the right correction is
    # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
    # Thankfully, shouldn't matter because our samples are pretty big usually anyways
Leo Gao's avatar
Leo Gao committed
476
    res = []
477
    chunk_size = min(1000, iters)
Leo Gao's avatar
Leo Gao committed
478
    from tqdm import tqdm
Fabrizio Milo's avatar
Fabrizio Milo committed
479

Leo Gao's avatar
Leo Gao committed
480
    print("bootstrapping for stddev:", f.__name__)
Fabrizio Milo's avatar
Fabrizio Milo committed
481
482
    for bootstrap in tqdm(
        pool.imap(
483
            _bootstrap_internal(f, chunk_size),
Fabrizio Milo's avatar
Fabrizio Milo committed
484
485
486
487
            [(i, xs) for i in range(iters // chunk_size)],
        ),
        total=iters // chunk_size,
    ):
Leo Gao's avatar
Leo Gao committed
488
        # sample w replacement
Leo Gao's avatar
Leo Gao committed
489
        res.extend(bootstrap)
Leo Gao's avatar
Leo Gao committed
490

Leo Gao's avatar
Leo Gao committed
491
    pool.close()
Leo Gao's avatar
Leo Gao committed
492
    return sample_stddev(res)
Leo Gao's avatar
Leo Gao committed
493
494


495
496
497
498
499
def stderr_for_metric(metric, bootstrap_iters: int):
    if bootstrap_iters <= 0:
        # return no function (don't compute stderr) if bootstrap iters = 0
        return None

500
501
502
503
504
505
506
507
    bootstrappable = [
        median,
        matthews_corrcoef,
        f1_score,
        perplexity,
        bleu,
        chrf,
        ter,
508
        nanmean,
509
510
511
512
513
514
515
516
    ]

    if metric in bootstrappable:
        return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters)

    stderr = {mean: mean_stderr, acc_all: acc_all_stderr}

    return stderr.get(metric, None)
517
518
519
520
521
522
523
524
525
526


def pooled_sample_stderr(stderrs: List[float], sizes: List[int]):
    # Used to aggregate bootstrapped stderrs across subtasks in a group,
    # when we are weighting by the size of each subtask.
    #

    assert len(stderrs) == len(sizes)

    # formula source: https://en.wikipedia.org/wiki/Pooled_variance
527
528
    # and: https://stats.stackexchange.com/a/4841331
    # this empirically seems to match running `stderr_for_metric` on all instances
529
530
    # from the subtasks concatenated with each other.
    pooled_sample_var = (
531
        sum([(size - 1) * stderr**2 * size for size, stderr in zip(sizes, stderrs)])
532
533
    ) / (sum(sizes) - len(sizes))

534
    return np.sqrt(pooled_sample_var / sum(sizes))
535
536
537


def combined_sample_stderr(stderrs: List[float], sizes: List[int], metrics=None):
Baber Abbasi's avatar
Baber Abbasi committed
538
539
540
    assert metrics is not None, (
        "Need to pass a list of each subtask's metric for this stderr aggregation"
    )
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
    assert len(stderrs) == len(sizes) and len(sizes) == len(metrics)

    # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1390 for more documentation.
    # This formula depends on sample means.
    # removed because it seems to give erroneously huge stderrs for groupings of tasks
    # and does not seem to match up with bootstrap-calculated stderrs for groups.

    ### don't use this unless a statistician has told you it's the right thing to do ###

    # accumulators: we'll aggregate pairwise N - 1 times
    variance = stderrs[0] ** 2
    curr_size = sizes[0]
    curr_score = metrics[0]

    for stderr, size, score in zip(stderrs[1:], sizes[1:], metrics[1:]):
        curr_score = ((curr_score * curr_size) + (score * size)) / (
            curr_size + size
        )  # NOTE: this assumes our aggregation fn is "mean"

        variance = ((curr_size - 1) * variance + (size - 1) * (stderr**2)) / (
            curr_size + size - 1
        ) + curr_size * size / ((curr_size + size) * (curr_size + size - 1)) * (
            curr_score - score
        ) ** 2

    return np.sqrt(variance)


def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True):
    # A helper function that is used to aggregate
    # subtask scores cross-task.
    # TODO: does not hold for non-mean aggregations
573
    if not weight_by_size:
574
575
576
577
        sizes = [1] * len(sizes)

    assert len(metrics) == len(sizes)

Lintang Sutawika's avatar
Lintang Sutawika committed
578
    return sum([metric * size for metric, size in zip(metrics, sizes)]) / sum(sizes)