"docs/vscode:/vscode.git/clone" did not exist on "57943630e24651e6d954b912e7fcdb2b4c719cc4"
Unverified Commit a15562e1 authored by Shijie Wu's avatar Shijie Wu Committed by GitHub
Browse files

Fix reference of import when called for the second time

parent 0287d264
......@@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try:
if 'pythainlp' not in sys.modules:
from pythainlp.tokenize import word_tokenize as th_word_tokenize
else:
th_word_tokenize = sys.modules['pythainlp'].word_tokenize
except (AttributeError, ImportError) as e:
logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
logger.error("1. pip install pythainlp")
......@@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try:
if 'jieba' not in sys.modules:
import jieba
else:
jieba = sys.modules['jieba']
except (AttributeError, ImportError) as e:
logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
logger.error("1. pip install jieba")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment