Unverified Commit 3cc23eee authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

More explicit error when failing to tensorize overflowing tokens (#5633)

parent b9d8af07
...@@ -511,6 +511,11 @@ class BatchEncoding(UserDict): ...@@ -511,6 +511,11 @@ class BatchEncoding(UserDict):
self[key] = tensor self[key] = tensor
except: # noqa E722 except: # noqa E722
if key == "overflowing_tokens":
raise ValueError(
"Unable to create tensor returning overflowing tokens of different lengths. "
"Please see if a fast version of this tokenizer is available to have this feature available."
)
raise ValueError( raise ValueError(
"Unable to create tensor, you should probably activate truncation and/or padding " "Unable to create tensor, you should probably activate truncation and/or padding "
"with 'padding=True' 'truncation=True' to have batched tensors with the same length." "with 'padding=True' 'truncation=True' to have batched tensors with the same length."
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment