Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ModelZoo
ResNet50_tensorflow
Commits
e4388f88
Commit
e4388f88
authored
Mar 17, 2021
by
A. Unique TensorFlower
Browse files
Update TODOs for {Bert,Sentencepiece}Tokenizer.get_config():
both are tracked in the same bug. PiperOrigin-RevId: 363431367
parent
45fd6cee
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
3 additions
and
19 deletions
+3
-19
official/nlp/modeling/layers/text_layers.py
official/nlp/modeling/layers/text_layers.py
+3
-19
No files found.
official/nlp/modeling/layers/text_layers.py
View file @
e4388f88
...
...
@@ -202,12 +202,7 @@ class BertTokenizer(tf.keras.layers.Layer):
def
get_config
(
self
):
# Skip in tf.saved_model.save(); fail if called direcly.
# TODO(arnoegw): Implement when switching to MutableHashTable, which gets
# initialized from the checkpoint and not from a vocab file.
# We cannot just put the original, user-supplied vocab file name into
# the config, because the path has to change as the SavedModel is copied
# around.
raise
NotImplementedError
(
"Not implemented yet."
)
raise
NotImplementedError
(
"TODO(b/170480226): implement"
)
def
get_special_tokens_dict
(
self
):
"""Returns dict of token ids, keyed by standard names for their purpose.
...
...
@@ -404,19 +399,8 @@ class SentencepieceTokenizer(tf.keras.layers.Layer):
return
_reshape
(
tokens
)
def
get_config
(
self
):
raise
NotImplementedError
(
"b/170480226"
)
# TODO(b/170480226): Uncomment and improve to fix the bug.
# config = {
# "model_serialized_proto": self._model_serialized_proto,
# "lower_case": self._lower_case,
# "tokenize_with_offsets": self.tokenize_with_offsets,
# "nbest_size": self._nbest_size,
# "alpha": self._alpha,
# "strip_diacritics": self._strip_diacritics,
# }
# base_config = super(SentencepieceTokenizer, self).get_config()
# base_config.update(config)
# return base_config
# Skip in tf.saved_model.save(); fail if called direcly.
raise
NotImplementedError
(
"TODO(b/170480226): implement"
)
def
get_special_tokens_dict
(
self
):
"""Returns dict of token ids, keyed by standard names for their purpose.
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment