"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "8e64ba2890bd3231916cddcec77ba6331c306031"
Unverified Commit 358478e7 authored by Stefan Schweter's avatar Stefan Schweter Committed by GitHub
Browse files

Examples: add Bloom support for token classification (#18632)

* examples: add Bloom support for token classification (FLAX, PyTorch and TensorFlow)

* examples: remove support for Bloom in token classication (FLAX and TensorFlow currently have no support for it)
parent 6d175c11
...@@ -348,7 +348,7 @@ def main(): ...@@ -348,7 +348,7 @@ def main():
) )
tokenizer_name_or_path = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path tokenizer_name_or_path = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path
if config.model_type in {"gpt2", "roberta"}: if config.model_type in {"bloom", "gpt2", "roberta"}:
tokenizer = AutoTokenizer.from_pretrained( tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name_or_path, tokenizer_name_or_path,
cache_dir=model_args.cache_dir, cache_dir=model_args.cache_dir,
......
...@@ -398,7 +398,7 @@ def main(): ...@@ -398,7 +398,7 @@ def main():
"You can do it from another script, save it, and load it from here, using --tokenizer_name." "You can do it from another script, save it, and load it from here, using --tokenizer_name."
) )
if config.model_type in {"gpt2", "roberta"}: if config.model_type in {"bloom", "gpt2", "roberta"}:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True, add_prefix_space=True) tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True, add_prefix_space=True)
else: else:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True) tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment