Unverified Commit a15562e1 authored by Shijie Wu's avatar Shijie Wu Committed by GitHub
Browse files

Fix reference of import when called for the second time

parent 0287d264
...@@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer): ...@@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try: try:
if 'pythainlp' not in sys.modules: if 'pythainlp' not in sys.modules:
from pythainlp.tokenize import word_tokenize as th_word_tokenize from pythainlp.tokenize import word_tokenize as th_word_tokenize
else:
th_word_tokenize = sys.modules['pythainlp'].word_tokenize
except (AttributeError, ImportError) as e: except (AttributeError, ImportError) as e:
logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps") logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
logger.error("1. pip install pythainlp") logger.error("1. pip install pythainlp")
...@@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer): ...@@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try: try:
if 'jieba' not in sys.modules: if 'jieba' not in sys.modules:
import jieba import jieba
else:
jieba = sys.modules['jieba']
except (AttributeError, ImportError) as e: except (AttributeError, ImportError) as e:
logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps") logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
logger.error("1. pip install jieba") logger.error("1. pip install jieba")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment