token[-1] -> token.rstrip('\n')

b1019d2a · Yiqing-Zhou · GitHub · bef0c629 · b1019d2a
Unverified Commit b1019d2a authored Jul 23, 2019 by Yiqing-Zhou Committed by GitHub Jul 23, 2019
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

pytorch_transformers/tokenization_bert.py pytorch_transformers/tokenization_bert.py +1 -1

No files found.
--- a/pytorch_transformers/tokenization_bert.py
+++ b/pytorch_transformers/tokenization_bert.py
@@ -69,7 +69,7 @@ def load_vocab(vocab_file):
    with open(vocab_file, "r", encoding="utf-8") as reader:
        tokens = reader.readlines()
    for index, token in enumerate(tokens):
-        token = token[:-1]
+        token = token.rstrip('\n')
        vocab[token] = index
    return vocab