Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
43fc77a2
Commit
43fc77a2
authored
Oct 24, 2020
by
Anish Thite
Browse files
make drop text consistent with gpt3 paper
parent
262fe250
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
6 additions
and
3 deletions
+6
-3
lm_eval/tasks/drop.py
lm_eval/tasks/drop.py
+6
-3
No files found.
lm_eval/tasks/drop.py
View file @
43fc77a2
...
@@ -10,6 +10,9 @@ from ..base import Dataset
...
@@ -10,6 +10,9 @@ from ..base import Dataset
class
DROP
(
Dataset
):
class
DROP
(
Dataset
):
DATAFOLDER
=
Path
(
__file__
).
parent
/
"../../data/drop"
DATAFOLDER
=
Path
(
__file__
).
parent
/
"../../data/drop"
def
__init__
(
self
):
self
.
download
()
def
has_training_docs
(
self
):
def
has_training_docs
(
self
):
"""Whether the task has a training set"""
"""Whether the task has a training set"""
return
True
return
True
...
@@ -35,10 +38,10 @@ class DROP(Dataset):
...
@@ -35,10 +38,10 @@ class DROP(Dataset):
pass
pass
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
def
doc_to_text
(
self
,
doc
,
include_target
=
True
):
doctext
=
"Passage: {}
\n
\n
"
.
format
(
doc
[
"passage"
])
doctext
=
"Passage: {}
\n
"
.
format
(
doc
[
"passage"
])
qa_texts
=
[]
qa_texts
=
[]
for
pair
in
doc
[
"qa_pairs"
]:
for
pair
in
doc
[
"qa_pairs"
]:
text
=
''
.
join
([
'Q: '
,
pair
[
'question'
],
'
\n
A: '
])
text
=
''
.
join
([
'Q
uestion
: '
,
pair
[
'question'
],
'
\n
A
nswer
: '
])
if
include_target
:
if
include_target
:
def
get_answer
(
ans_dict
):
def
get_answer
(
ans_dict
):
if
ans_dict
[
'number'
]
!=
''
:
if
ans_dict
[
'number'
]
!=
''
:
...
@@ -52,7 +55,7 @@ class DROP(Dataset):
...
@@ -52,7 +55,7 @@ class DROP(Dataset):
ans_dict
[
'date'
][
'year'
]]).
strip
()
ans_dict
[
'date'
][
'year'
]]).
strip
()
text
=
''
.
join
([
text
,
get_answer
(
pair
[
'answer'
])])
text
=
''
.
join
([
text
,
get_answer
(
pair
[
'answer'
])])
qa_texts
.
append
(
text
)
qa_texts
.
append
(
text
)
return
''
.
join
([
doctext
,
'
\n
\n
'
.
join
(
qa_texts
)])
return
''
.
join
([
doctext
,
'
\n
'
.
join
(
qa_texts
)])
def
evaluate
(
self
,
docs
,
lm
,
provide_description
,
num_fewshot
):
def
evaluate
(
self
,
docs
,
lm
,
provide_description
,
num_fewshot
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment