Skip to content

GitLab

Commit 0a694910 authored by lintangsutawika's avatar lintangsutawika
Browse files

update

parent 61c49449
Hide whitespace changes
Inline Side-by-side
Showing with 7 additions and 11 deletions
lm_eval/api/task.py
View file @ 0a694910
...@@ -1029,18 +1029,8 @@ class ConfigurableTask(Task): ...@@ -1029,18 +1029,8 @@ class ConfigurableTask(Task):
choices = self.doc_to_choice(doc) choices = self.doc_to_choice(doc)
gold = choices[gold] gold = choices[gold]
if type(gold) is int:
choices = self.doc_to_choice(doc)
gold = choices[gold]
for key, result in zip(self._metric_fn_list.keys(), results): for key, result in zip(self._metric_fn_list.keys(), results):
if not self.multiple_target: if self.multiple_target:
result = self._metric_fn_list[key](
references=[gold],
predictions=[result],
**self._metric_fn_kwargs[key],
)
else:
# in the case where we have multiple targets, # in the case where we have multiple targets,
# return true if any are true # return true if any are true
# TODO: this may break for multipLe_target, non zero-or-1 metrics # TODO: this may break for multipLe_target, non zero-or-1 metrics
...@@ -1059,6 +1049,12 @@ class ConfigurableTask(Task): ...@@ -1059,6 +1049,12 @@ class ConfigurableTask(Task):
result = 1.0 result = 1.0
else: else:
result = 0.0 result = 0.0
else:
result = self._metric_fn_list[key](
references=[gold],
predictions=[result],
**self._metric_fn_kwargs[key],
)
if isinstance(result, dict): if isinstance(result, dict):
result_dict.update(result) result_dict.update(result)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment