Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
15579e2d
Commit
15579e2d
authored
Jan 21, 2020
by
Lysandre
Browse files
[SQuAD v2] Code quality
parent
088fa7b7
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
13 additions
and
6 deletions
+13
-6
src/transformers/data/processors/squad.py
src/transformers/data/processors/squad.py
+13
-6
No files found.
src/transformers/data/processors/squad.py
View file @
15579e2d
...
@@ -242,7 +242,7 @@ def squad_convert_example_to_features(example, max_seq_length, doc_stride, max_q
...
@@ -242,7 +242,7 @@ def squad_convert_example_to_features(example, max_seq_length, doc_stride, max_q
token_to_orig_map
=
span
[
"token_to_orig_map"
],
token_to_orig_map
=
span
[
"token_to_orig_map"
],
start_position
=
start_position
,
start_position
=
start_position
,
end_position
=
end_position
,
end_position
=
end_position
,
is_impossible
=
span_is_impossible
is_impossible
=
span_is_impossible
,
)
)
)
)
return
features
return
features
...
@@ -351,7 +351,7 @@ def squad_convert_examples_to_features(
...
@@ -351,7 +351,7 @@ def squad_convert_examples_to_features(
all_end_positions
,
all_end_positions
,
all_cls_index
,
all_cls_index
,
all_p_mask
,
all_p_mask
,
all_is_impossible
all_is_impossible
,
)
)
return
features
,
dataset
return
features
,
dataset
...
@@ -372,7 +372,7 @@ def squad_convert_examples_to_features(
...
@@ -372,7 +372,7 @@ def squad_convert_examples_to_features(
"end_position"
:
ex
.
end_position
,
"end_position"
:
ex
.
end_position
,
"cls_index"
:
ex
.
cls_index
,
"cls_index"
:
ex
.
cls_index
,
"p_mask"
:
ex
.
p_mask
,
"p_mask"
:
ex
.
p_mask
,
"is_impossible"
:
ex
.
is_impossible
"is_impossible"
:
ex
.
is_impossible
,
},
},
)
)
...
@@ -380,7 +380,13 @@ def squad_convert_examples_to_features(
...
@@ -380,7 +380,13 @@ def squad_convert_examples_to_features(
gen
,
gen
,
(
(
{
"input_ids"
:
tf
.
int32
,
"attention_mask"
:
tf
.
int32
,
"token_type_ids"
:
tf
.
int32
},
{
"input_ids"
:
tf
.
int32
,
"attention_mask"
:
tf
.
int32
,
"token_type_ids"
:
tf
.
int32
},
{
"start_position"
:
tf
.
int64
,
"end_position"
:
tf
.
int64
,
"cls_index"
:
tf
.
int64
,
"p_mask"
:
tf
.
int32
,
"is_impossible"
:
tf
.
int32
},
{
"start_position"
:
tf
.
int64
,
"end_position"
:
tf
.
int64
,
"cls_index"
:
tf
.
int64
,
"p_mask"
:
tf
.
int32
,
"is_impossible"
:
tf
.
int32
,
},
),
),
(
(
{
{
...
@@ -393,7 +399,7 @@ def squad_convert_examples_to_features(
...
@@ -393,7 +399,7 @@ def squad_convert_examples_to_features(
"end_position"
:
tf
.
TensorShape
([]),
"end_position"
:
tf
.
TensorShape
([]),
"cls_index"
:
tf
.
TensorShape
([]),
"cls_index"
:
tf
.
TensorShape
([]),
"p_mask"
:
tf
.
TensorShape
([
None
]),
"p_mask"
:
tf
.
TensorShape
([
None
]),
"is_impossible"
:
tf
.
TensorShape
([])
"is_impossible"
:
tf
.
TensorShape
([])
,
},
},
),
),
)
)
...
@@ -663,7 +669,7 @@ class SquadFeatures(object):
...
@@ -663,7 +669,7 @@ class SquadFeatures(object):
token_to_orig_map
,
token_to_orig_map
,
start_position
,
start_position
,
end_position
,
end_position
,
is_impossible
is_impossible
,
):
):
self
.
input_ids
=
input_ids
self
.
input_ids
=
input_ids
self
.
attention_mask
=
attention_mask
self
.
attention_mask
=
attention_mask
...
@@ -682,6 +688,7 @@ class SquadFeatures(object):
...
@@ -682,6 +688,7 @@ class SquadFeatures(object):
self
.
end_position
=
end_position
self
.
end_position
=
end_position
self
.
is_impossible
=
is_impossible
self
.
is_impossible
=
is_impossible
class
SquadResult
(
object
):
class
SquadResult
(
object
):
"""
"""
Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset.
Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset.
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment