Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
894c0697
Commit
894c0697
authored
Feb 12, 2022
by
Stephen Hogg
Browse files
Name changes as per leogao2
parent
96f3e5b3
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
16 additions
and
6 deletions
+16
-6
lm_eval/tasks/qasper.py
lm_eval/tasks/qasper.py
+16
-6
No files found.
lm_eval/tasks/qasper.py
View file @
894c0697
...
@@ -172,25 +172,30 @@ class QASPER(HFTask):
...
@@ -172,25 +172,30 @@ class QASPER(HFTask):
# Handle unanswerability first
# Handle unanswerability first
unanswerable_gold
=
doc
[
"answer_type"
]
==
"unanswerable"
unanswerable_gold
=
doc
[
"answer_type"
]
==
"unanswerable"
unanswerable_pred
=
exp
(
logprob_unanswerable
)
>
1
-
exp
(
logprob_unanswerable
)
unanswerable_pred
=
exp
(
logprob_unanswerable
)
>
1
-
exp
(
logprob_unanswerable
)
res_dict
[
"f1_un"
]
=
(
unanswerable_gold
,
unanswerable_pred
)
res_dict
[
"f1_un
answerable
"
]
=
(
unanswerable_gold
,
unanswerable_pred
)
# Handle yes/no questions
# Handle yes/no questions
if
doc
[
"answer_type"
]
==
"bool"
:
if
doc
[
"answer_type"
]
==
"bool"
:
gold
=
1
if
doc
[
"answer"
]
==
"yes"
else
0
gold
=
1
if
doc
[
"answer"
]
==
"yes"
else
0
pred
=
ll_yes
>
ll_no
pred
=
ll_yes
>
ll_no
res_dict
[
"f1_y
n
"
]
=
(
gold
,
pred
)
res_dict
[
"f1_y
esno
"
]
=
(
gold
,
pred
)
# Handle completions
# Handle completions
if
doc
[
"answer_type"
]
==
"free form answer"
:
if
doc
[
"answer_type"
]
==
"free form answer"
:
res_dict
[
"f1_ab"
]
=
token_f1_score
(
res
,
doc
[
"answer"
])
res_dict
[
"f1_ab
stractive
"
]
=
token_f1_score
(
res
,
doc
[
"answer"
])
# Handle extraction
# Handle extraction
if
doc
[
"answer_type"
]
==
"extractive_spans"
:
if
doc
[
"answer_type"
]
==
"extractive_spans"
:
res_dict
[
"f1_ex"
]
=
0
res_dict
[
"f1_ex
tractive
"
]
=
0
return
res_dict
return
res_dict
def
aggregation
(
self
):
def
aggregation
(
self
):
return
{
"f1_un"
:
f1_score
,
"f1_yn"
:
f1_score
,
"f1_ab"
:
mean
,
"f1_ex"
:
mean
}
return
{
"f1_unanswerable"
:
f1_score
,
"f1_yesno"
:
f1_score
,
"f1_abstractive"
:
mean
,
"f1_extractive"
:
mean
,
}
def
construct_requests
(
self
,
doc
,
ctx
):
def
construct_requests
(
self
,
doc
,
ctx
):
"""Uses RequestFactory to construct Requests and returns an iterable of
"""Uses RequestFactory to construct Requests and returns an iterable of
...
@@ -219,4 +224,9 @@ class QASPER(HFTask):
...
@@ -219,4 +224,9 @@ class QASPER(HFTask):
A dictionary where keys are the names of submetrics and values are
A dictionary where keys are the names of submetrics and values are
whether a higher value of the submetric is better
whether a higher value of the submetric is better
"""
"""
return
{
"f1_un"
:
True
,
"f1_yn"
:
True
,
"f1_ab"
:
True
,
"f1_ex"
:
True
}
return
{
"f1_unanswerable"
:
True
,
"f1_yesno"
:
True
,
"f1_abstractive"
:
True
,
"f1_extractive"
:
True
,
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment