tokenization updates

pytorch_pretrained_bert/tokenization.py

@@ -135,9 +135,10 @@ class BertTokenizer(object):
         return tokens
 
     def save_vocabulary(self, vocab_path):
-        """Save the tokenizer vocabulary to a path."""
+        """Save the tokenizer vocabulary to a directory or file."""
         index = 0
-        vocab_file = os.path.join(vocab_path, VOCAB_NAME)
+        if os.path.isdir(vocab_path):
+            vocab_file = os.path.join(vocab_path, VOCAB_NAME)
         with open(vocab_file, "w", encoding="utf-8") as writer:
             for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
                 if index != token_index:

pytorch_pretrained_bert/tokenization_transfo_xl.py

@@ -145,8 +145,10 @@ class TransfoXLTokenizer(object):
             raise ValueError('No <unkown> token in vocabulary')
 
     def save_vocabulary(self, vocab_path):
+        """Save the tokenizer vocabulary to a directory or file."""
         index = 0
-        vocab_file = os.path.join(vocab_path, VOCAB_NAME)
+        if os.path.isdir(vocab_path):
+            vocab_file = os.path.join(vocab_path, VOCAB_NAME)
         torch.save(self.__dict__, vocab_file)
         return vocab_file