add debug log (#2757)

74332955 · Baber Abbasi · GitHub · 8b5c5c13 · 74332955
Unverified Commit 74332955 authored Mar 04, 2025 by Baber Abbasi Committed by GitHub Mar 04, 2025
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 0 deletions

lm_eval/api/task.py lm_eval/api/task.py +4 -0

No files found.
--- a/lm_eval/api/task.py
+++ b/lm_eval/api/task.py
@@ -838,6 +838,10 @@ class ConfigurableTask(Task):
                filter_pipeline = build_filter_ensemble(filter_name, components)
                self._filters.append(filter_pipeline)
        else:
+            # TODO: handle repeats in a more general way rather than just discarding
+            eval_logger.debug(
+                "No custom filters defined. Using default 'take_first' filter for handling repeats."
+            )
            self._filters = [build_filter_ensemble("none", [["take_first", None]])]

        if self.config.use_prompt is not None: