Some models have no tokenizers (#11064)

eb3479e7 · Lysandre Debut · GitHub · 773e4c72 · eb3479e7
Unverified Commit eb3479e7 authored Apr 05, 2021 by Lysandre Debut Committed by GitHub Apr 05, 2021
Show whitespace changes
Inline Side-by-side

Showing with 8 additions and 7 deletions

tests/test_tokenization_common.py tests/test_tokenization_common.py +8 -7

No files found.
--- a/tests/test_tokenization_common.py
+++ b/tests/test_tokenization_common.py
@@ -69,6 +69,7 @@ def merge_model_tokenizer_mappings(
    model_tokenizer_mapping = OrderedDict([])

    for configuration in configurations:
+        if configuration in model_mapping and configuration in tokenizer_mapping:
            model = model_mapping[configuration]
            tokenizer = tokenizer_mapping[configuration][0]
            tokenizer_fast = tokenizer_mapping[configuration][1]