Commit cddbf9f6 authored by lintangsutawika's avatar lintangsutawika
Browse files

reformat

parent 65e5bc69
...@@ -1022,28 +1022,30 @@ class ConfigurableTask(Task): ...@@ -1022,28 +1022,30 @@ class ConfigurableTask(Task):
gold_index_error = True gold_index_error = True
else: else:
if type(gold) is int: if type(gold) is int:
gold = gold if i < len(choice) else -1000 gold = gold if gold < len(choices) else -1000
elif type(gold) is str: elif type(gold) is str:
gold = choices.index(gold) if gold in choices else -1000 gold = choices.index(gold) if gold in choices else -1000
if gold == -1000: if gold == -1000:
gold_index_error = True gold_index_error = True
if gold_index_error: if gold_index_error:
eval_logger.warning( eval_logger.warning(
f"Label index was not in within range of availabel choices," f"Label index was not in within range of available choices,"
f"Sample:\n\n{doc}\n\n" f"Sample:\n\n{doc}\n\n"
) )
if self.multiple_target: if self.multiple_target:
acc = 1.0 if pred in gold else 0.0 acc = 1.0 if pred in gold else 0.0
acc_norm = 1.0 if pred_norm in gold else 0.0 acc_norm = 1.0 if pred_norm in gold else 0.0
exact_match = int(any([is_greedy[i] if i != -1000 else 0 for i in gold])) exact_match = int(
any([is_greedy[i] if i != -1000 else 0 for i in gold])
)
else: else:
acc = 1.0 if pred == gold else 0.0 acc = 1.0 if pred == gold else 0.0
acc_norm = 1.0 if pred_norm == gold else 0.0 acc_norm = 1.0 if pred_norm == gold else 0.0
# TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly
exact_match = int(is_greedy[gold]) if i != -1000 else 0 exact_match = int(is_greedy[gold]) if gold != -1000 else 0
result_dict = { result_dict = {
**({"acc": acc} if "acc" in use_metric else {}), **({"acc": acc} if "acc" in use_metric else {}),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment