metrics.py 2.71 KB
Newer Older
&'s avatar
& committed
1
import math
Leo Gao's avatar
Leo Gao committed
2
import random
&'s avatar
& committed
3
4


Leo Gao's avatar
Leo Gao committed
5
def pop_stddev(arr):
6
    mu = sum(arr) / len(arr)
Leo Gao's avatar
Leo Gao committed
7
8
9
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))


Leo Gao's avatar
Leo Gao committed
10
def sample_stddev(arr):
11
    mu = sum(arr) / len(arr)
Leo Gao's avatar
Leo Gao committed
12
13
14
    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))


Leo Gao's avatar
Leo Gao committed
15
def mean_stderr(arr):
Leo Gao's avatar
Leo Gao committed
16
    return sample_stddev(arr) / math.sqrt(len(arr))
Leo Gao's avatar
Leo Gao committed
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


def acc_all_stderr(items):
    # Only count as correct if all answers are labeled correctly for each question
    question_scoring_dict = {}
    preds = list(zip(*items))[0]
    docs = list(zip(*items))[1]

    for doc, pred in zip(docs, preds):
        question_id = doc["idx"]["question"]
        if question_id not in question_scoring_dict:
            question_scoring_dict[question_id] = []

        gold_label = doc["label"] == 1
        question_scoring_dict[question_id].append(gold_label == pred)

    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
    return acc

&'s avatar
& committed
36
37
38
39
40
41
42
43
44
45

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    """Compute max metric between prediction and each ground truth."""
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)


Leo Gao's avatar
Leo Gao committed
46
47
48
49
class _bootstrap_internal:
    def __init__(self, f, n):
        self.f = f
        self.n = n
50

Leo Gao's avatar
Leo Gao committed
51
52
53
54
55
56
57
58
59
    def __call__(self, v):
        i, xs = v
        rnd = random.Random()
        rnd.seed(i)
        res = []
        for _ in range(self.n):
            res.append(self.f(rnd.choices(xs, k=len(xs))))
        return res

Leo Gao's avatar
Leo Gao committed
60

61
def bootstrap_stderr(f, xs, iters):
Leo Gao's avatar
Leo Gao committed
62
    import multiprocessing as mp
Fabrizio Milo's avatar
Fabrizio Milo committed
63

Leo Gao's avatar
Leo Gao committed
64
    pool = mp.Pool(mp.cpu_count())
Leo Gao's avatar
Leo Gao committed
65
    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
Fabrizio Milo's avatar
Fabrizio Milo committed
66
    # equivalent to stderr calculated without Bessel's correction in the stddev.
Leo Gao's avatar
Leo Gao committed
67
68
69
70
    # Unfortunately, I haven't been able to figure out what the right correction is
    # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
    # Thankfully, shouldn't matter because our samples are pretty big usually anyways
Leo Gao's avatar
Leo Gao committed
71
    res = []
72
    chunk_size = min(1000, iters)
Leo Gao's avatar
Leo Gao committed
73
    from tqdm import tqdm
Fabrizio Milo's avatar
Fabrizio Milo committed
74

Leo Gao's avatar
Leo Gao committed
75
    print("bootstrapping for stddev:", f.__name__)
Fabrizio Milo's avatar
Fabrizio Milo committed
76
77
    for bootstrap in tqdm(
        pool.imap(
78
            _bootstrap_internal(f, chunk_size),
Fabrizio Milo's avatar
Fabrizio Milo committed
79
80
81
82
            [(i, xs) for i in range(iters // chunk_size)],
        ),
        total=iters // chunk_size,
    ):
Leo Gao's avatar
Leo Gao committed
83
        # sample w replacement
Leo Gao's avatar
Leo Gao committed
84
        res.extend(bootstrap)
Leo Gao's avatar
Leo Gao committed
85

Leo Gao's avatar
Leo Gao committed
86
    pool.close()
Leo Gao's avatar
Leo Gao committed
87
    return sample_stddev(res)
Leo Gao's avatar
Leo Gao committed
88
89


Jonathan Tow's avatar
Jonathan Tow committed
90
91
def yesno(x):
    if x:
Fabrizio Milo's avatar
Fabrizio Milo committed
92
        return "yes"
Jonathan Tow's avatar
Jonathan Tow committed
93
    else:
Fabrizio Milo's avatar
Fabrizio Milo committed
94
        return "no"