Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
cddbf9f6
Commit
cddbf9f6
authored
Aug 30, 2023
by
lintangsutawika
Browse files
reformat
parent
65e5bc69
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
8 additions
and
6 deletions
+8
-6
lm_eval/api/task.py
lm_eval/api/task.py
+8
-6
No files found.
lm_eval/api/task.py
View file @
cddbf9f6
...
@@ -1022,28 +1022,30 @@ class ConfigurableTask(Task):
...
@@ -1022,28 +1022,30 @@ class ConfigurableTask(Task):
gold_index_error
=
True
gold_index_error
=
True
else
:
else
:
if
type
(
gold
)
is
int
:
if
type
(
gold
)
is
int
:
gold
=
gold
if
i
<
len
(
choice
)
else
-
1000
gold
=
gold
if
gold
<
len
(
choice
s
)
else
-
1000
elif
type
(
gold
)
is
str
:
elif
type
(
gold
)
is
str
:
gold
=
choices
.
index
(
gold
)
if
gold
in
choices
else
-
1000
gold
=
choices
.
index
(
gold
)
if
gold
in
choices
else
-
1000
if
gold
==
-
1000
:
if
gold
==
-
1000
:
gold_index_error
=
True
gold_index_error
=
True
if
gold_index_error
:
if
gold_index_error
:
eval_logger
.
warning
(
eval_logger
.
warning
(
f
"Label index was not in within range of availab
e
l choices,"
f
"Label index was not in within range of availabl
e
choices,"
f
"Sample:
\n\n
{
doc
}
\n\n
"
f
"Sample:
\n\n
{
doc
}
\n\n
"
)
)
if
self
.
multiple_target
:
if
self
.
multiple_target
:
acc
=
1.0
if
pred
in
gold
else
0.0
acc
=
1.0
if
pred
in
gold
else
0.0
acc_norm
=
1.0
if
pred_norm
in
gold
else
0.0
acc_norm
=
1.0
if
pred_norm
in
gold
else
0.0
exact_match
=
int
(
any
([
is_greedy
[
i
]
if
i
!=
-
1000
else
0
for
i
in
gold
]))
exact_match
=
int
(
any
([
is_greedy
[
i
]
if
i
!=
-
1000
else
0
for
i
in
gold
])
)
else
:
else
:
acc
=
1.0
if
pred
==
gold
else
0.0
acc
=
1.0
if
pred
==
gold
else
0.0
acc_norm
=
1.0
if
pred_norm
==
gold
else
0.0
acc_norm
=
1.0
if
pred_norm
==
gold
else
0.0
# TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly
# TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly
exact_match
=
int
(
is_greedy
[
gold
])
if
i
!=
-
1000
else
0
exact_match
=
int
(
is_greedy
[
gold
])
if
gold
!=
-
1000
else
0
result_dict
=
{
result_dict
=
{
**
({
"acc"
:
acc
}
if
"acc"
in
use_metric
else
{}),
**
({
"acc"
:
acc
}
if
"acc"
in
use_metric
else
{}),
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment