Unverified Commit 2e3452af authored by Ripose's avatar Ripose Committed by GitHub
Browse files

make sentencepiece import conditional in bertjapanesetokenizer (#20012)

parent 8827e1b2
...@@ -21,11 +21,14 @@ import os ...@@ -21,11 +21,14 @@ import os
import unicodedata import unicodedata
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
import sentencepiece as spm
from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging from ...utils import is_sentencepiece_available, logging
if is_sentencepiece_available():
import sentencepiece as spm
else:
spm = None
logger = logging.get_logger(__name__) logger = logging.get_logger(__name__)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment