calculate std if items > 1

91279a6e · lintangsutawika · f8c2cfcb · 91279a6e
Commit 91279a6e authored Dec 12, 2023 by lintangsutawika
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

lm_eval/evaluator.py lm_eval/evaluator.py +3 -1

No files found.
--- a/lm_eval/evaluator.py
+++ b/lm_eval/evaluator.py
@@ -492,8 +492,10 @@ def evaluate(
                    else bootstrap_iters,
                )

-                if stderr is not None:
+                if stderr is not None and len(items) > 1:
                    results[task_name][metric + "_stderr" + "," + key] = stderr(items)
+                else:
+                    results[task_name][metric + "_stderr" + "," + key] = "N/A"

        if bool(results):
            for group, task_list in reversed(task_hierarchy.items()):