metrics.py 9.17 KB
Newer Older
lintangsutawika's avatar
lintangsutawika committed
1
import logging
&'s avatar
& committed
2
import math
lintangsutawika's avatar
lintangsutawika committed
3
import random
4
5
from collections.abc import Iterable

lintangsutawika's avatar
lintangsutawika committed
6
import evaluate
7
8
9
import numpy as np
import sacrebleu
import sklearn.metrics
&'s avatar
& committed
10

lintangsutawika's avatar
lintangsutawika committed
11
from lm_eval.api.registry import register_metric
12

lintangsutawika's avatar
lintangsutawika committed
13

14
eval_logger = logging.getLogger("lm-eval")
15

16

17
18
19
20
21
22
23
24
def mean(arr):
    return sum(arr) / len(arr)


def median(arr):
    return arr[len(arr) // 2]


lintangsutawika's avatar
lintangsutawika committed
25
26
27
28
29
def weighted_mean(items):
    a, b = zip(*items)
    return sum(a) / sum(b)


30
31
32
33
34
@register_metric(
    metric="perplexity",
    higher_is_better=False,
    output_type="loglikelihood",
)
lintangsutawika's avatar
lintangsutawika committed
35
36
def perplexity(items):
    return math.exp(-mean(items))
37

lintangsutawika's avatar
lintangsutawika committed
38
39
40
41
42
43
44
45

@register_metric(
    metric=["word_perplexity", "byte_perplexity"],
    higher_is_better=False,
    output_type="loglikelihood_rolling",
)
def weighted_perplexity(items):  # This is a passthrough function
    return math.exp(-weighted_mean(items))
46
47


48
@register_metric(
lintangsutawika's avatar
lintangsutawika committed
49
50
51
    metric="bits_per_byte",
    higher_is_better=False,
    output_type="loglikelihood_rolling",
52
)
lintangsutawika's avatar
lintangsutawika committed
53
54
def bits_per_byte(items):
    return -weighted_mean(items) / math.log(2)
55
56


lintangsutawika's avatar
lintangsutawika committed
57
58
59
60
61
@register_metric(
    metric="f1",
    higher_is_better=True,
    output_type="multiple_choice",
)
haileyschoelkopf's avatar
haileyschoelkopf committed
62
63
64
65
66
67
68
69
70
def f1_score(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    fscore = sklearn.metrics.f1_score(golds, preds)

    return np.max(fscore)


lintangsutawika's avatar
lintangsutawika committed
71
72
73
74
75
@register_metric(
    metric="mcc",
    higher_is_better=True,
    output_type="multiple_choice",
)
haileyschoelkopf's avatar
haileyschoelkopf committed
76
77
78
79
80
81
82
def matthews_corrcoef(items):
    unzipped_list = list(zip(*items))
    golds = unzipped_list[0]
    preds = unzipped_list[1]
    return sklearn.metrics.matthews_corrcoef(golds, preds)


lintangsutawika's avatar
lintangsutawika committed
83
84
85
86
87
@register_metric(
    metric="bleu",
    higher_is_better=True,
    output_type="generate_until",
)
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
def bleu(items):
    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
    for evaluating a generated sentence to a reference sentence. It counts matching
    n-grams in the candidate translation to n-grams in the reference text, where
    1-gram or unigram would be each token and a bigram comparison would be each
    word pair. The comparison is made regardless of word order
    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
    Paper: https://www.aclweb.org/anthology/P02-1040/

    Higher is better
    """
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
    refs, preds = _sacreformat(refs, preds)
    return sacrebleu.corpus_bleu(preds, refs).score


lintangsutawika's avatar
lintangsutawika committed
105
106
107
108
109
@register_metric(
    metric="chrf",
    higher_is_better=True,
    output_type="generate_until",
)
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def chrf(items):
    """chrF++ is a tool for automatic evaluation of machine translation output
    based on character n-gram precision and recall enhanced with word n-grams.
    Source: https://github.com/m-popovic/chrF
    Paper: https://www.aclweb.org/anthology/W15-3049.pdf

    Higher is better  # TODO I think
    """
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
    refs, preds = _sacreformat(refs, preds)
    return sacrebleu.corpus_chrf(preds, refs).score


lintangsutawika's avatar
lintangsutawika committed
124
125
126
127
128
@register_metric(
    metric="ter",
    higher_is_better=True,
    output_type="generate_until",
)
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def ter(items):
    """Translation Error Rate is an error metric for machine translation that
    measures the number of edits required to change a system output into one
    of the references
    Source: http://www.cs.umd.edu/~snover/tercom/
    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf

    Lower is better
    """
    refs = list(zip(*items))[0]
    preds = list(zip(*items))[1]
    refs, preds = _sacreformat(refs, preds)
    return sacrebleu.corpus_ter(preds, refs).score


144
@register_metric(
lintangsutawika's avatar
lintangsutawika committed
145
146
147
    metric=["acc", "acc_norm"],
    higher_is_better=True,
    output_type=["loglikelihood", "multiple_choice"],
148
)
lintangsutawika's avatar
lintangsutawika committed
149
150
def aggregate_acc_fn(items):
    return mean(items)
151
152
153


@register_metric(
lintangsutawika's avatar
lintangsutawika committed
154
155
156
    metric="acc_mutual_info",
    higher_is_better=True,
    output_type="multiple_choice",
157
)
lintangsutawika's avatar
lintangsutawika committed
158
159
160
def acc_mutual_info_fn(items):
    return mean(items)

161

lintangsutawika's avatar
lintangsutawika committed
162
exact_match = evaluate.load("exact_match")
163
164
165


@register_metric(
lintangsutawika's avatar
lintangsutawika committed
166
167
168
    metric="exact_match",
    higher_is_better=True,
    output_type="generate_until",
169
)
lintangsutawika's avatar
lintangsutawika committed
170
171
def exact_match_fn(**kwargs):
    return exact_match.compute(**kwargs)
172

&'s avatar
& committed
173

Leo Gao's avatar
Leo Gao committed
174
def pop_stddev(arr):
175
    mu = mean(arr)
Leo Gao's avatar
Leo Gao committed
176
177
178
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))


Leo Gao's avatar
Leo Gao committed
179
def sample_stddev(arr):
180
    mu = mean(arr)
Leo Gao's avatar
Leo Gao committed
181
182
183
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))


Leo Gao's avatar
Leo Gao committed
184
def mean_stderr(arr):
Leo Gao's avatar
Leo Gao committed
185
    return sample_stddev(arr) / math.sqrt(len(arr))
Leo Gao's avatar
Leo Gao committed
186
187


lintangsutawika's avatar
lintangsutawika committed
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
@register_metric(
    metric="acc_all",
    higher_is_better=True,
    output_type="loglikelihood",
)
def acc_all(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        paragraph_id = doc["idx"]["paragraph"]
        question_id = doc["idx"]["question"]
        if (paragraph_id, question_id) not in question_scoring_dict:
            question_scoring_dict[(paragraph_id, question_id)] = []

        gold_label = doc["label"] == 1

        question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred)
    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
    return acc
210
211


Leo Gao's avatar
Leo Gao committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
def acc_all_stderr(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
    return acc

&'s avatar
& committed
229
230
231
232
233
234
235
236
237
238

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
def is_non_str_iterable(obj):
    return isinstance(obj, Iterable) and not isinstance(obj, str)


def _sacreformat(refs, preds):
    """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
    # Sacrebleu expects (List[str], List[List[str])
    #   e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])

    # Note [ref1_stream] is the first reference for each pred.
    # So lists are size N and (M, N) for N preds and M possible refs for each pred
    # This is a different order of dimensions that I would expect

    # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
    # Must become List[List[str]] with the inner list corresponding to preds
    if not is_non_str_iterable(refs):
        refs = list(refs)
    if not is_non_str_iterable(refs[0]):
        refs = [[ref] for ref in refs]
    refs = list(zip(*refs))
    # Note the number of refs in each ref list much match the number of preds

    # We expect preds to be List[str] or List[List[str]]. Must become List[str]
    if not is_non_str_iterable(preds):
        preds = list(preds)
    if is_non_str_iterable(preds[0]):
        assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
        preds = [pred[0] for pred in preds]

    return refs, preds


# stderr stuff


Leo Gao's avatar
Leo Gao committed
274
class _bootstrap_internal:
Ethan Smith's avatar
Ethan Smith committed
275
    def __init__(self, f, n) -> None:
Leo Gao's avatar
Leo Gao committed
276
277
        self.f = f
        self.n = n
278

Leo Gao's avatar
Leo Gao committed
279
280
281
282
283
284
285
286
287
    def __call__(self, v):
        i, xs = v
        rnd = random.Random()
        rnd.seed(i)
        res = []
        for _ in range(self.n):
            res.append(self.f(rnd.choices(xs, k=len(xs))))
        return res

Leo Gao's avatar
Leo Gao committed
288

289
def bootstrap_stderr(f, xs, iters):
Leo Gao's avatar
Leo Gao committed
290
    import multiprocessing as mp
Fabrizio Milo's avatar
Fabrizio Milo committed
291

Leo Gao's avatar
Leo Gao committed
292
    pool = mp.Pool(mp.cpu_count())
Leo Gao's avatar
Leo Gao committed
293
    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
Fabrizio Milo's avatar
Fabrizio Milo committed
294
    # equivalent to stderr calculated without Bessel's correction in the stddev.
Leo Gao's avatar
Leo Gao committed
295
296
297
298
    # Unfortunately, I haven't been able to figure out what the right correction is
    # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
    # Thankfully, shouldn't matter because our samples are pretty big usually anyways
Leo Gao's avatar
Leo Gao committed
299
    res = []
300
    chunk_size = min(1000, iters)
Leo Gao's avatar
Leo Gao committed
301
    from tqdm import tqdm
Fabrizio Milo's avatar
Fabrizio Milo committed
302

Leo Gao's avatar
Leo Gao committed
303
    print("bootstrapping for stddev:", f.__name__)
Fabrizio Milo's avatar
Fabrizio Milo committed
304
305
    for bootstrap in tqdm(
        pool.imap(
306
            _bootstrap_internal(f, chunk_size),
Fabrizio Milo's avatar
Fabrizio Milo committed
307
308
309
310
            [(i, xs) for i in range(iters // chunk_size)],
        ),
        total=iters // chunk_size,
    ):
Leo Gao's avatar
Leo Gao committed
311
        # sample w replacement
Leo Gao's avatar
Leo Gao committed
312
        res.extend(bootstrap)
Leo Gao's avatar
Leo Gao committed
313

Leo Gao's avatar
Leo Gao committed
314
    pool.close()
Leo Gao's avatar
Leo Gao committed
315
    return sample_stddev(res)
Leo Gao's avatar
Leo Gao committed
316
317


318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
def stderr_for_metric(metric, bootstrap_iters):
    bootstrappable = [
        median,
        matthews_corrcoef,
        f1_score,
        perplexity,
        bleu,
        chrf,
        ter,
    ]

    if metric in bootstrappable:
        return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters)

    stderr = {mean: mean_stderr, acc_all: acc_all_stderr}

    return stderr.get(metric, None)