Skip to content

GitLab

"...resnet50_tensorflow.git" did not exist on "45848c446ae8300ac598f743f8ba35b225074fad"
Commit 21aa92d2 authored by haileyschoelkopf's avatar haileyschoelkopf
Browse files

rename var

parent 40f81c42
Hide whitespace changes
Inline Side-by-side
Showing with 14 additions and 12 deletions
lm_eval/api/task.py
View file @ 21aa92d2
...@@ -1011,34 +1011,36 @@ class ConfigurableTask(Task): ...@@ -1011,34 +1011,36 @@ class ConfigurableTask(Task):
scores = [] scores = []
for gold_option in gold: for gold_option in gold:
try: try:
res = self._metric_fn_list[key]( result_score = self._metric_fn_list[key](
references=[gold_option], references=[gold_option],
predictions=[result], predictions=[result],
**self._metric_fn_kwargs[key], **self._metric_fn_kwargs[key],
) )
except TypeError: # TODO: this is hacky and I don't want to do it except TypeError: # TODO: this is hacky and I don't want to do it
result = self._metric_fn_list[key]([gold_option, result]) result_score = self._metric_fn_list[key](
if isinstance(res, dict): [gold_option, result]
)
if isinstance(result_score, dict):
# TODO: this handles the case where HF evaluate returns a dict. # TODO: this handles the case where HF evaluate returns a dict.
res = res[key] result_score = result_score[key]
scores.append(res) scores.append(result_score)
if any(scores): if any(scores):
result = 1.0 result_score = 1.0
else: else:
result = 0.0 result_score = 0.0
else: else:
try: try:
result = self._metric_fn_list[key]( result_score = self._metric_fn_list[key](
references=[gold], references=[gold],
predictions=[result], predictions=[result],
**self._metric_fn_kwargs[key], **self._metric_fn_kwargs[key],
) )
except TypeError: except TypeError:
result = self._metric_fn_list[key]([gold, result]) result_score = self._metric_fn_list[key]([gold, result])
if isinstance(result, dict): if isinstance(result_score, dict):
result_dict.update(result) result_dict.update(result_score)
else: else:
result_dict[key] = result result_dict[key] = result_score
else: else:
raise ValueError( raise ValueError(
f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ", f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment