limit fix (#1785)

cee785e0 · KonradSzafer · GitHub · 18f4eb57 · cee785e0
Unverified Commit cee785e0 authored May 05, 2024 by KonradSzafer Committed by GitHub May 05, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 1 deletion

lm_eval/evaluator.py lm_eval/evaluator.py +4 -1

No files found.
--- a/lm_eval/evaluator.py
+++ b/lm_eval/evaluator.py
@@ -592,7 +592,10 @@ def evaluate(
            "n-samples": {
                task_output.task_name: {
                    "original": len(task_output.task.eval_docs),
-                    "effective": min(limit, len(task_output.task.eval_docs)),
+                    "effective": min(
+                        limit if limit else len(task_output.task.eval_docs),
+                        len(task_output.task.eval_docs),
+                    ),
                }
                for task_output in eval_tasks
            },