multiple_output option for greedy_until

4c363ef8 · lintangsutawika · f09f3ab8 · 4c363ef8
Commit 4c363ef8 authored Jul 11, 2023 by lintangsutawika
Show whitespace changes
Inline Side-by-side

Showing with 5 additions and 5 deletions

lm_eval/api/task.py lm_eval/api/task.py +5 -5

No files found.
--- a/lm_eval/api/task.py
+++ b/lm_eval/api/task.py
@@ -998,7 +998,7 @@ class ConfigurableTask(Task):

            for key, result in zip(self._metric_fn_list.keys(), results):
                _dict = self._metric_fn_list[key](
-                    references=[gold],
+                    references=gold if self.multiple_output else [gold],
                    predictions=[result],
                    **self._metric_fn_kwargs[key],
                )