Merge pull request #2081 from pglock/patch-1

handle string with only whitespaces as empty

transformers/tokenization_utils.py

@@ -669,7 +669,7 @@ class PreTrainedTokenizer(object):
             return result
 
         def split_on_tokens(tok_list, text):
-            if not text:
+            if not text.strip():
                 return []
             if not tok_list:
                 return self._tokenize(text, **kwargs)