Skip to content

GitLab

"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "d751dbecb267e4e3ae77352e97a6470da93a0094"
Unverified Commit 2aa9c2f2 authored by Patrick von Platen's avatar Patrick von Platen Committed by GitHub
Browse files

fix blenderbot tok (#9532)

parent 406cbf58
Hide whitespace changes
Inline Side-by-side
Showing with 8 additions and 6 deletions
src/transformers/models/blenderbot/tokenization_blenderbot.py
View file @ 2aa9c2f2
...@@ -48,9 +48,11 @@ class BlenderbotTokenizer(RobertaTokenizer): ...@@ -48,9 +48,11 @@ class BlenderbotTokenizer(RobertaTokenizer):
"tokenizer_config_file": "tokenizer_config.json", "tokenizer_config_file": "tokenizer_config.json",
} }
pretrained_vocab_files_map = { pretrained_vocab_files_map = {
"vocab_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/vocab.json"}, "vocab_file": {CKPT_3B: "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
"merges_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/merges.txt"}, "merges_file": {CKPT_3B: "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
"tokenizer_config_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/tokenizer_config.json"}, "tokenizer_config_file": {
CKPT_3B: "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
},
} }
max_model_input_sizes = {"facebook/blenderbot-3B": 128} max_model_input_sizes = {"facebook/blenderbot-3B": 128}
......
src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
View file @ 2aa9c2f2
...@@ -82,13 +82,13 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer): ...@@ -82,13 +82,13 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
} }
pretrained_vocab_files_map = { pretrained_vocab_files_map = {
"vocab_file": { "vocab_file": {
"facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/blob/main/vocab.json" "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"
}, },
"merges_file": { "merges_file": {
"facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/blob/main/merges.txt" "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"
}, },
"tokenizer_config_file": { "tokenizer_config_file": {
"facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/blob/main/tokenizer.json" "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer.json"
}, },
} }
max_model_input_sizes = {"facebook/blenderbot_small-90M": 512} max_model_input_sizes = {"facebook/blenderbot_small-90M": 512}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment