Commit 6e2c28a1 authored by Lysandre's avatar Lysandre
Browse files

Run SQuAD warning when the doc stride may be too high

parent b8f43cb2
...@@ -671,6 +671,13 @@ def main(): ...@@ -671,6 +671,13 @@ def main():
parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features") parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
args = parser.parse_args() args = parser.parse_args()
if args.doc_stride >= args.max_seq_length - args.max_query_length:
logger.warning(
"WARNING - You've set a doc stride which may be superior to the document length in some "
"examples. This could result in errors when building features from the examples. Please reduce the doc "
"stride or increase the maximum length to ensure the features are correctly built."
)
if ( if (
os.path.exists(args.output_dir) os.path.exists(args.output_dir)
and os.listdir(args.output_dir) and os.listdir(args.output_dir)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment