"git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "6cbfa7bf4cb41864eca43a553a6d831ed39e8af0"
Unverified Commit a15562e1 authored by Shijie Wu's avatar Shijie Wu Committed by GitHub
Browse files

Fix reference of import when called for the second time

parent 0287d264
...@@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer): ...@@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try: try:
if 'pythainlp' not in sys.modules: if 'pythainlp' not in sys.modules:
from pythainlp.tokenize import word_tokenize as th_word_tokenize from pythainlp.tokenize import word_tokenize as th_word_tokenize
else:
th_word_tokenize = sys.modules['pythainlp'].word_tokenize
except (AttributeError, ImportError) as e: except (AttributeError, ImportError) as e:
logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps") logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
logger.error("1. pip install pythainlp") logger.error("1. pip install pythainlp")
...@@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer): ...@@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try: try:
if 'jieba' not in sys.modules: if 'jieba' not in sys.modules:
import jieba import jieba
else:
jieba = sys.modules['jieba']
except (AttributeError, ImportError) as e: except (AttributeError, ImportError) as e:
logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps") logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
logger.error("1. pip install jieba") logger.error("1. pip install jieba")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment