Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
5960874b
Unverified
Commit
5960874b
authored
Feb 09, 2021
by
Leo Gao
Committed by
GitHub
Feb 09, 2021
Browse files
Update qa4mre.py
parent
a264f449
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
3 deletions
+4
-3
lm_eval/tasks/qa4mre.py
lm_eval/tasks/qa4mre.py
+4
-3
No files found.
lm_eval/tasks/qa4mre.py
View file @
5960874b
...
...
@@ -64,9 +64,10 @@ class QA4MRE(MultipleChoiceTask):
tree
=
ET
.
parse
(
textfilename
)
root
=
tree
.
getroot
()
# TODO: context is much larger than the context sometimes
# at the moment, it just gets left-truncated by LM automatically, and maybe that's good enough?
for
reading_test
in
root
.
iter
(
'reading-test'
):
src
=
reading_test
[
0
].
text
src
=
src
.
r
strip
(
"
\n\t\t\t
"
).
replace
(
"
\'
"
,
"'"
)
src
=
src
.
strip
().
replace
(
"
\'
"
,
"'"
)
for
qid
,
question
in
enumerate
(
reading_test
.
iter
(
'q'
)):
out_doc
=
self
.
_convert_standard
(
question
)
out_doc
[
'source'
]
=
src
...
...
@@ -79,7 +80,7 @@ class QA4MRE(MultipleChoiceTask):
return
self
.
load_docs
(
f
"data/qa4mre/QA4MRE-
{
self
.
YEAR
}
-EN_GS.xml"
)
def
doc_to_text
(
self
,
doc
):
return
"
{}
\n
{}
"
.
format
(
doc
[
"source"
],
doc
[
"query"
])
return
"{}
\n
Question: {}
\n
Answer:
"
.
format
(
doc
[
"source"
],
doc
[
"query"
])
class
QA4MRE_2011
(
QA4MRE
):
YEAR
=
2011
...
...
@@ -88,4 +89,4 @@ class QA4MRE_2012(QA4MRE):
YEAR
=
2012
class
QA4MRE_2013
(
QA4MRE
):
YEAR
=
2013
\ No newline at end of file
YEAR
=
2013
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment