"tests/vscode:/vscode.git/clone" did not exist on "9b6385488617ee99188b60c0525cbe4ec7e8f2a0"
Commit e4388f88 authored by A. Unique TensorFlower's avatar A. Unique TensorFlower
Browse files

Update TODOs for {Bert,Sentencepiece}Tokenizer.get_config():

both are tracked in the same bug.

PiperOrigin-RevId: 363431367
parent 45fd6cee
...@@ -202,12 +202,7 @@ class BertTokenizer(tf.keras.layers.Layer): ...@@ -202,12 +202,7 @@ class BertTokenizer(tf.keras.layers.Layer):
def get_config(self): def get_config(self):
# Skip in tf.saved_model.save(); fail if called direcly. # Skip in tf.saved_model.save(); fail if called direcly.
# TODO(arnoegw): Implement when switching to MutableHashTable, which gets raise NotImplementedError("TODO(b/170480226): implement")
# initialized from the checkpoint and not from a vocab file.
# We cannot just put the original, user-supplied vocab file name into
# the config, because the path has to change as the SavedModel is copied
# around.
raise NotImplementedError("Not implemented yet.")
def get_special_tokens_dict(self): def get_special_tokens_dict(self):
"""Returns dict of token ids, keyed by standard names for their purpose. """Returns dict of token ids, keyed by standard names for their purpose.
...@@ -404,19 +399,8 @@ class SentencepieceTokenizer(tf.keras.layers.Layer): ...@@ -404,19 +399,8 @@ class SentencepieceTokenizer(tf.keras.layers.Layer):
return _reshape(tokens) return _reshape(tokens)
def get_config(self): def get_config(self):
raise NotImplementedError("b/170480226") # Skip in tf.saved_model.save(); fail if called direcly.
# TODO(b/170480226): Uncomment and improve to fix the bug. raise NotImplementedError("TODO(b/170480226): implement")
# config = {
# "model_serialized_proto": self._model_serialized_proto,
# "lower_case": self._lower_case,
# "tokenize_with_offsets": self.tokenize_with_offsets,
# "nbest_size": self._nbest_size,
# "alpha": self._alpha,
# "strip_diacritics": self._strip_diacritics,
# }
# base_config = super(SentencepieceTokenizer, self).get_config()
# base_config.update(config)
# return base_config
def get_special_tokens_dict(self): def get_special_tokens_dict(self):
"""Returns dict of token ids, keyed by standard names for their purpose. """Returns dict of token ids, keyed by standard names for their purpose.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment