fix_tokenizer_snapshot_download_bug (#4493)

6ad58f42 · fuchen.ljl · GitHub · dd1a50a8 · 6ad58f42
Unverified Commit 6ad58f42 authored May 01, 2024 by fuchen.ljl Committed by GitHub Apr 30, 2024
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

vllm/transformers_utils/tokenizer.py vllm/transformers_utils/tokenizer.py +1 -1

No files found.
--- a/vllm/transformers_utils/tokenizer.py
+++ b/vllm/transformers_utils/tokenizer.py
@@ -79,7 +79,7 @@ def get_tokenizer(
                revision=revision,
                local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
                # Ignore weights - we only need the tokenizer.
-                ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
+                ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
            tokenizer_name = tokenizer_path

    if tokenizer_mode == "slow":