"git@developer.sourcefind.cn:chenpangpang/ComfyUI.git" did not exist on "23507882d21937e68688dfe7bc7eb2ed94c2f1a7"
Unverified Commit 9eb3a410 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Remove deprecated arguments from new run_clm (#8197)

parent 00112c35
...@@ -259,14 +259,14 @@ def main(): ...@@ -259,14 +259,14 @@ def main():
) )
if data_args.block_size <= 0: if data_args.block_size <= 0:
block_size = tokenizer.max_len block_size = tokenizer.model_max_length
else: else:
if data_args.block_size > tokenizer.max_len: if data_args.block_size > tokenizer.model_max_length:
logger.warn( logger.warn(
f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model" f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model"
f"({tokenizer.max_len}). Using block_size={tokenizer.max_len}." f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
) )
block_size = min(data_args.block_size, tokenizer.max_len) block_size = min(data_args.block_size, tokenizer.model_max_length)
# Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
def group_texts(examples): def group_texts(examples):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment