Use sample stddev

lm_eval/metrics.py

@@ -12,14 +12,18 @@ def mean(arr):
     return sum(arr) / len(arr)
 
 
-def stddev(arr):
+def pop_stddev(arr):
     mu = mean(arr)
     return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))
 
 
+def sample_stddev(arr):
+    mu = mean(arr)
+    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))
+
+
 def mean_stderr(arr):
-    print(stddev(arr), len(arr))
-    return stddev(arr) / math.sqrt(len(arr))
+    return sample_stddev(arr) / math.sqrt(len(arr))
 
 
 def median(arr):
@@ -181,7 +185,7 @@ def bootstrap_stderr(f, xs, iters=10000):
         bootstrap = f(rnd.choices(xs, k=len(xs)))
         res.append(bootstrap)
 
-    return stddev(res)
+    return sample_stddev(res)
 
 
 def stderr_for_metric(metric):

tests/test_misc.py

@@ -10,3 +10,11 @@ def test_bootstrapping():
     bootstrapped = metrics.bootstrap_stderr(metrics.mean, arr, iters=100000)
 
     assert bootstrapped == pytest.approx(expected, abs=1e-4)
+
+
+def test_bootstrapping_stella():
+    arr = [0.1, 0.3, 0.2, 0.25, 0.3, 0.1, 0.22]
+    expected = metrics.mean_stderr(arr)
+    bootstrapped = metrics.bootstrap_stderr(metrics.mean, arr, iters=100000)
+
+    assert bootstrapped == pytest.approx(expected, abs=1e-5)
\ No newline at end of file