Unverified Commit 3b9a1dc1 authored by Stas Bekman's avatar Stas Bekman Committed by GitHub
Browse files

[examples] improve block_size warning message (#21463)

parent 4435c7f5
...@@ -459,8 +459,9 @@ def main(): ...@@ -459,8 +459,9 @@ def main():
block_size = tokenizer.model_max_length block_size = tokenizer.model_max_length
if block_size > 1024: if block_size > 1024:
logger.warning( logger.warning(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
"Picking 1024 instead. You can change that default value by passing --block_size xxx." " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
" override this default with `--block_size xxx`."
) )
block_size = 1024 block_size = 1024
else: else:
......
...@@ -407,8 +407,9 @@ def main(): ...@@ -407,8 +407,9 @@ def main():
block_size = tokenizer.model_max_length block_size = tokenizer.model_max_length
if block_size > 1024: if block_size > 1024:
logger.warning( logger.warning(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
"Picking 1024 instead. You can change that default value by passing --block_size xxx." " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
" override this default with `--block_size xxx`."
) )
block_size = 1024 block_size = 1024
else: else:
......
...@@ -414,8 +414,9 @@ def main(): ...@@ -414,8 +414,9 @@ def main():
max_seq_length = tokenizer.model_max_length max_seq_length = tokenizer.model_max_length
if max_seq_length > 1024: if max_seq_length > 1024:
logger.warning( logger.warning(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
"Picking 1024 instead. You can change that default value by passing --max_seq_length xxx." " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
" override this default with `--block_size xxx`."
) )
max_seq_length = 1024 max_seq_length = 1024
else: else:
......
...@@ -399,8 +399,9 @@ def main(): ...@@ -399,8 +399,9 @@ def main():
max_seq_length = tokenizer.model_max_length max_seq_length = tokenizer.model_max_length
if max_seq_length > 1024: if max_seq_length > 1024:
logger.warning( logger.warning(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
"Picking 1024 instead. You can change that default value by passing --max_seq_length xxx." " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
" override this default with `--block_size xxx`."
) )
max_seq_length = 1024 max_seq_length = 1024
else: else:
......
...@@ -336,8 +336,9 @@ def main(): ...@@ -336,8 +336,9 @@ def main():
max_seq_length = tokenizer.model_max_length max_seq_length = tokenizer.model_max_length
if max_seq_length > 1024: if max_seq_length > 1024:
logger.warning( logger.warning(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
"Picking 1024 instead. You can change that default value by passing --max_seq_length xxx." " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
" override this default with `--block_size xxx`."
) )
max_seq_length = 1024 max_seq_length = 1024
else: else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment