Unverified Commit 1efb21c7 authored by skumar951's avatar skumar951 Committed by GitHub
Browse files

Explicitly check if token ID's are None in TFBertTokenizer constructor (#28824)

Add an explicit none-check, since token ids can be 0
parent 721ee783
...@@ -91,9 +91,9 @@ class TFBertTokenizer(keras.layers.Layer): ...@@ -91,9 +91,9 @@ class TFBertTokenizer(keras.layers.Layer):
self.vocab_list = vocab_list self.vocab_list = vocab_list
self.do_lower_case = do_lower_case self.do_lower_case = do_lower_case
self.cls_token_id = cls_token_id or vocab_list.index("[CLS]") self.cls_token_id = vocab_list.index("[CLS]") if cls_token_id is None else cls_token_id
self.sep_token_id = sep_token_id or vocab_list.index("[SEP]") self.sep_token_id = vocab_list.index("[SEP]") if sep_token_id is None else sep_token_id
self.pad_token_id = pad_token_id or vocab_list.index("[PAD]") self.pad_token_id = vocab_list.index("[PAD]") if pad_token_id is None else pad_token_id
self.paired_trimmer = ShrinkLongestTrimmer(max_length - 3, axis=1) # Allow room for special tokens self.paired_trimmer = ShrinkLongestTrimmer(max_length - 3, axis=1) # Allow room for special tokens
self.max_length = max_length self.max_length = max_length
self.padding = padding self.padding = padding
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment