Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
33508ae3
Commit
33508ae3
authored
Dec 04, 2019
by
LysandreJik
Browse files
Remove `only_first`
parent
f7e4a7cd
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
5 additions
and
8 deletions
+5
-8
transformers/data/processors/squad.py
transformers/data/processors/squad.py
+5
-8
No files found.
transformers/data/processors/squad.py
View file @
33508ae3
...
@@ -300,29 +300,29 @@ class SquadProcessor(DataProcessor):
...
@@ -300,29 +300,29 @@ class SquadProcessor(DataProcessor):
return
examples
return
examples
def
get_train_examples
(
self
,
data_dir
,
only_first
=
None
):
def
get_train_examples
(
self
,
data_dir
):
"""See base class."""
"""See base class."""
if
self
.
train_file
is
None
:
if
self
.
train_file
is
None
:
raise
ValueError
(
"SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor"
)
raise
ValueError
(
"SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor"
)
with
open
(
os
.
path
.
join
(
data_dir
,
self
.
train_file
),
"r"
,
encoding
=
'utf-8'
)
as
reader
:
with
open
(
os
.
path
.
join
(
data_dir
,
self
.
train_file
),
"r"
,
encoding
=
'utf-8'
)
as
reader
:
input_data
=
json
.
load
(
reader
)[
"data"
]
input_data
=
json
.
load
(
reader
)[
"data"
]
return
self
.
_create_examples
(
input_data
,
"train"
,
only_first
)
return
self
.
_create_examples
(
input_data
,
"train"
)
def
get_dev_examples
(
self
,
data_dir
,
only_first
=
None
):
def
get_dev_examples
(
self
,
data_dir
):
"""See base class."""
"""See base class."""
if
self
.
dev_file
is
None
:
if
self
.
dev_file
is
None
:
raise
ValueError
(
"SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor"
)
raise
ValueError
(
"SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor"
)
with
open
(
os
.
path
.
join
(
data_dir
,
self
.
dev_file
),
"r"
,
encoding
=
'utf-8'
)
as
reader
:
with
open
(
os
.
path
.
join
(
data_dir
,
self
.
dev_file
),
"r"
,
encoding
=
'utf-8'
)
as
reader
:
input_data
=
json
.
load
(
reader
)[
"data"
]
input_data
=
json
.
load
(
reader
)[
"data"
]
return
self
.
_create_examples
(
input_data
,
"dev"
,
only_first
)
return
self
.
_create_examples
(
input_data
,
"dev"
)
def
get_labels
(
self
):
def
get_labels
(
self
):
"""See base class."""
"""See base class."""
return
[
"0"
,
"1"
]
return
[
"0"
,
"1"
]
def
_create_examples
(
self
,
input_data
,
set_type
,
only_first
=
None
):
def
_create_examples
(
self
,
input_data
,
set_type
):
"""Creates examples for the training and dev sets."""
"""Creates examples for the training and dev sets."""
is_training
=
set_type
==
"train"
is_training
=
set_type
==
"train"
...
@@ -363,9 +363,6 @@ class SquadProcessor(DataProcessor):
...
@@ -363,9 +363,6 @@ class SquadProcessor(DataProcessor):
)
)
examples
.
append
(
example
)
examples
.
append
(
example
)
if
only_first
is
not
None
and
len
(
examples
)
>
only_first
:
return
examples
return
examples
return
examples
class
SquadV1Processor
(
SquadProcessor
):
class
SquadV1Processor
(
SquadProcessor
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment