Unverified Commit 21451ec6 authored by Philipp Glock's avatar Philipp Glock Committed by GitHub
Browse files

handle string with only whitespaces as empty

parent 35ff345f
...@@ -634,7 +634,7 @@ class PreTrainedTokenizer(object): ...@@ -634,7 +634,7 @@ class PreTrainedTokenizer(object):
return result return result
def split_on_tokens(tok_list, text): def split_on_tokens(tok_list, text):
if not text: if not text.strip():
return [] return []
if not tok_list: if not tok_list:
return self._tokenize(text, **kwargs) return self._tokenize(text, **kwargs)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment