Skip to content

GitLab

"examples/vscode:/vscode.git/clone" did not exist on "32d2031458b34fd3073d5fcac99a4124913dc8a1"
Commit 865d4d59 authored by Julien Chaumond's avatar Julien Chaumond
Browse files

[ci] Close #4481

parent a3af8e86
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 3 deletions
tests/test_tokenization_bert_japanese.py
View file @ 865d4d59
......@@ -26,7 +26,7 @@ from transformers.tokenization_bert_japanese import (
)
from .test_tokenization_common import TokenizerTesterMixin
from .utils import custom_tokenizers, slow
from .utils import custom_tokenizers
@custom_tokenizers
......@@ -126,7 +126,6 @@ class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"), ["こん", "##ばんは", "[UNK]", "こんにちは"])
@slow
def test_sequence_builders(self):
tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese")
......@@ -141,6 +140,7 @@ class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
assert encoded_pair == [2] + text + [3] + text_2 + [3]
@custom_tokenizers
class BertJapaneseCharacterTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = BertJapaneseTokenizer
......@@ -187,7 +187,6 @@ class BertJapaneseCharacterTokenizationTest(TokenizerTesterMixin, unittest.TestC
self.assertListEqual(tokenizer.tokenize("こんにちほ"), ["こ", "ん", "に", "ち", "[UNK]"])
@slow
def test_sequence_builders(self):
tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese-char")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment