Unverified Commit 98308586 authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Fix the `bert-base-cased` tokenizer configuration test (#29105)

Fix test
parent 593230f0
...@@ -223,7 +223,7 @@ class AutoTokenizerTest(unittest.TestCase): ...@@ -223,7 +223,7 @@ class AutoTokenizerTest(unittest.TestCase):
config = get_tokenizer_config("google-bert/bert-base-cased") config = get_tokenizer_config("google-bert/bert-base-cased")
_ = config.pop("_commit_hash", None) _ = config.pop("_commit_hash", None)
# If we ever update google-bert/bert-base-cased tokenizer config, this dict here will need to be updated. # If we ever update google-bert/bert-base-cased tokenizer config, this dict here will need to be updated.
self.assertEqual(config, {"do_lower_case": False}) self.assertEqual(config, {"do_lower_case": False, "model_max_length": 512})
# This model does not have a tokenizer_config so we get back an empty dict. # This model does not have a tokenizer_config so we get back an empty dict.
config = get_tokenizer_config(SMALL_MODEL_IDENTIFIER) config = get_tokenizer_config(SMALL_MODEL_IDENTIFIER)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment