fix `tokenizer_type` to avoid error when loading checkpoint back (#20062)

19067711 · Sourab Mangrulkar · GitHub · 3502c202 · 19067711
Unverified Commit 19067711 authored Nov 04, 2022 by Sourab Mangrulkar Committed by GitHub Nov 04, 2022
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

src/transformers/models/megatron_gpt2/checkpoint_reshaping_and_interoperability.py ...egatron_gpt2/checkpoint_reshaping_and_interoperability.py +1 -1

No files found.
--- a/src/transformers/models/megatron_gpt2/checkpoint_reshaping_and_interoperability.py
+++ b/src/transformers/models/megatron_gpt2/checkpoint_reshaping_and_interoperability.py
@@ -648,7 +648,7 @@ def convert_checkpoint_from_transformers_to_megatron(args):
        "data_parallel_size": args.target_data_parallel_size,
        "make_vocab_size_divisible_by": args.make_vocab_size_divisible_by,
        "rank": 0,
-        "tokenizer_type": None,
+        "tokenizer_type": "GPT2BPETokenizer",
    }

    if config.activation_function == "gelu":