Add test for Chinese tokenization

4d124baf · Yaser Martinez Palenzuela · GitHub · 0ce2f496 · 4d124baf
Unverified Commit 4d124baf authored Nov 05, 2018 by Yaser Martinez Palenzuela Committed by GitHub Nov 05, 2018
Hide whitespace changes
Inline Side-by-side

Showing with 7 additions and 0 deletions

tests/tokenization_test.py tests/tokenization_test.py +7 -0

No files found.
--- a/tests/tokenization_test.py
+++ b/tests/tokenization_test.py
@@ -43,6 +43,13 @@ class TokenizationTest(unittest.TestCase):
        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9])

+    def test_chinese(self):
+        tokenizer = tokenization.BasicTokenizer()
+    
+        self.assertListEqual(
+            tokenizer.tokenize(u"ah\u535A\u63A8zz"),
+            [u"ah", u"\u535A", u"\u63A8", u"zz"])  
+
    def test_basic_tokenizer_lower(self):
        tokenizer = tokenization.BasicTokenizer(do_lower_case=True)