Unverified Commit d2ffc3fc authored by Alexander Ljungberg's avatar Alexander Ljungberg Committed by GitHub
Browse files

Fix a mistake in Llama weight converter log output. (#22764)

Fixed string format; better tokenizer message.

Before: `Saving a {tokenizer_class} to {tokenizer_path}`
After: `Saving a LlamaTokenizerFast to outdir.`
parent 9af845af
...@@ -104,7 +104,7 @@ def write_model(model_path, input_base_path, model_size): ...@@ -104,7 +104,7 @@ def write_model(model_path, input_base_path, model_size):
print(f"Fetching all parameters from the checkpoint at {input_base_path}.") print(f"Fetching all parameters from the checkpoint at {input_base_path}.")
# Load weights # Load weights
if model_size == "7B": if model_size == "7B":
# Not shared # Not sharded
# (The sharded implementation would also work, but this is simpler.) # (The sharded implementation would also work, but this is simpler.)
loaded = torch.load(os.path.join(input_base_path, "consolidated.00.pth"), map_location="cpu") loaded = torch.load(os.path.join(input_base_path, "consolidated.00.pth"), map_location="cpu")
else: else:
...@@ -244,7 +244,7 @@ def write_model(model_path, input_base_path, model_size): ...@@ -244,7 +244,7 @@ def write_model(model_path, input_base_path, model_size):
def write_tokenizer(tokenizer_path, input_tokenizer_path): def write_tokenizer(tokenizer_path, input_tokenizer_path):
# Initialize the tokenizer based on the `spm` model # Initialize the tokenizer based on the `spm` model
tokenizer_class = LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast tokenizer_class = LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
print("Saving a {tokenizer_class} to {tokenizer_path}") print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}.")
tokenizer = tokenizer_class(input_tokenizer_path) tokenizer = tokenizer_class(input_tokenizer_path)
tokenizer.save_pretrained(tokenizer_path) tokenizer.save_pretrained(tokenizer_path)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment