Unverified Commit d7379477 authored by maru0kun's avatar maru0kun Committed by GitHub
Browse files

Fix typo

parent 0b52642d
...@@ -174,7 +174,7 @@ class PreTrainedTokenizer(object): ...@@ -174,7 +174,7 @@ class PreTrainedTokenizer(object):
return self.convert_tokens_to_ids(self._eos_token) return self.convert_tokens_to_ids(self._eos_token)
@property @property
def unk_token_is(self): def unk_token_id(self):
""" Id of the unknown token in the vocabulary. Log an error if used while not having been set. """ """ Id of the unknown token in the vocabulary. Log an error if used while not having been set. """
if self._unk_token is None: if self._unk_token is None:
logger.error("Using unk_token, but it is not set yet.") logger.error("Using unk_token, but it is not set yet.")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment