Skip to content

GitLab

Unverified Commit d0b5ed11 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Harder check for IndexErrors in QA scripts (#15438) 

* Harder check for IndexErrors in QA scripts

* Make test stronger
parent 8e5d4e49
Hide whitespace changes
Inline Side-by-side
Showing with 9 additions and 0 deletions
examples/flax/question-answering/utils_qa.py
View file @ d0b5ed11
...@@ -137,7 +137,9 @@ def postprocess_qa_predictions( ...@@ -137,7 +137,9 @@ def postprocess_qa_predictions(
start_index >= len(offset_mapping) start_index >= len(offset_mapping)
or end_index >= len(offset_mapping) or end_index >= len(offset_mapping)
or offset_mapping[start_index] is None or offset_mapping[start_index] is None
or len(offset_mapping[start_index]) < 2
or offset_mapping[end_index] is None or offset_mapping[end_index] is None
or len(offset_mapping[end_index]) < 2
): ):
continue continue
# Don't consider answers with a length that is either < 0 or > max_answer_length. # Don't consider answers with a length that is either < 0 or > max_answer_length.
...@@ -147,6 +149,7 @@ def postprocess_qa_predictions( ...@@ -147,6 +149,7 @@ def postprocess_qa_predictions(
# provided). # provided).
if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False): if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):
continue continue
prelim_predictions.append( prelim_predictions.append(
{ {
"offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]), "offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]),
......
examples/pytorch/question-answering/utils_qa.py
View file @ d0b5ed11
...@@ -137,7 +137,9 @@ def postprocess_qa_predictions( ...@@ -137,7 +137,9 @@ def postprocess_qa_predictions(
start_index >= len(offset_mapping) start_index >= len(offset_mapping)
or end_index >= len(offset_mapping) or end_index >= len(offset_mapping)
or offset_mapping[start_index] is None or offset_mapping[start_index] is None
or len(offset_mapping[start_index]) < 2
or offset_mapping[end_index] is None or offset_mapping[end_index] is None
or len(offset_mapping[end_index]) < 2
): ):
continue continue
# Don't consider answers with a length that is either < 0 or > max_answer_length. # Don't consider answers with a length that is either < 0 or > max_answer_length.
...@@ -147,6 +149,7 @@ def postprocess_qa_predictions( ...@@ -147,6 +149,7 @@ def postprocess_qa_predictions(
# provided). # provided).
if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False): if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):
continue continue
prelim_predictions.append( prelim_predictions.append(
{ {
"offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]), "offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]),
......
examples/tensorflow/question-answering/utils_qa.py
View file @ d0b5ed11
...@@ -137,7 +137,9 @@ def postprocess_qa_predictions( ...@@ -137,7 +137,9 @@ def postprocess_qa_predictions(
start_index >= len(offset_mapping) start_index >= len(offset_mapping)
or end_index >= len(offset_mapping) or end_index >= len(offset_mapping)
or offset_mapping[start_index] is None or offset_mapping[start_index] is None
or len(offset_mapping[start_index]) < 2
or offset_mapping[end_index] is None or offset_mapping[end_index] is None
or len(offset_mapping[end_index]) < 2
): ):
continue continue
# Don't consider answers with a length that is either < 0 or > max_answer_length. # Don't consider answers with a length that is either < 0 or > max_answer_length.
...@@ -147,6 +149,7 @@ def postprocess_qa_predictions( ...@@ -147,6 +149,7 @@ def postprocess_qa_predictions(
# provided). # provided).
if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False): if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):
continue continue
prelim_predictions.append( prelim_predictions.append(
{ {
"offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]), "offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment