"docs/vscode:/vscode.git/clone" did not exist on "7a0cf0ec936864c693b47aefc63472f09bccfe06"
Commit 6e2c28a1 authored by Lysandre's avatar Lysandre
Browse files

Run SQuAD warning when the doc stride may be too high

parent b8f43cb2
......@@ -671,6 +671,13 @@ def main():
parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
args = parser.parse_args()
if args.doc_stride >= args.max_seq_length - args.max_query_length:
logger.warning(
"WARNING - You've set a doc stride which may be superior to the document length in some "
"examples. This could result in errors when building features from the examples. Please reduce the doc "
"stride or increase the maximum length to ensure the features are correctly built."
)
if (
os.path.exists(args.output_dir)
and os.listdir(args.output_dir)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment