Unverified Commit 06033798 authored by metacryptom's avatar metacryptom Committed by GitHub
Browse files

fix wrong using getattr to get dict value (#232)

parent 665c4896
...@@ -22,7 +22,7 @@ def get_tokenizer( ...@@ -22,7 +22,7 @@ def get_tokenizer(
logger.info( logger.info(
"OpenLLaMA models do not support the fast tokenizer. " "OpenLLaMA models do not support the fast tokenizer. "
"Using the slow tokenizer instead.") "Using the slow tokenizer instead.")
elif config.model_type == "llama" and getattr(kwargs, "use_fast", True): elif config.model_type == "llama" and kwargs.get("use_fast", True):
# LLaMA fast tokenizer causes protobuf errors in some environments. # LLaMA fast tokenizer causes protobuf errors in some environments.
# However, we found that the below LLaMA fast tokenizer works well in # However, we found that the below LLaMA fast tokenizer works well in
# most environments. # most environments.
...@@ -31,7 +31,7 @@ def get_tokenizer( ...@@ -31,7 +31,7 @@ def get_tokenizer(
f"Using the LLaMA fast tokenizer in '{model_name}' to avoid " f"Using the LLaMA fast tokenizer in '{model_name}' to avoid "
"potential protobuf errors.") "potential protobuf errors.")
elif config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER: elif config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
if getattr(kwargs, "use_fast", False) == True: if kwargs.get("use_fast", False) == True:
raise ValueError( raise ValueError(
f"Cannot use the fast tokenizer for {config.model_type} due to " f"Cannot use the fast tokenizer for {config.model_type} due to "
"bugs in the fast tokenizer.") "bugs in the fast tokenizer.")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment