Commit 5d912e7e authored by Julien Chaumond's avatar Julien Chaumond
Browse files

Tweak typing for #3566

parent 94eb68d7
...@@ -19,6 +19,7 @@ import collections ...@@ -19,6 +19,7 @@ import collections
import logging import logging
import os import os
import unicodedata import unicodedata
from typing import Optional
from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer, load_vocab from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer, load_vocab
...@@ -165,10 +166,10 @@ class BertJapaneseTokenizer(BertTokenizer): ...@@ -165,10 +166,10 @@ class BertJapaneseTokenizer(BertTokenizer):
return split_tokens return split_tokens
class MecabTokenizer(object): class MecabTokenizer:
"""Runs basic tokenization with MeCab morphological parser.""" """Runs basic tokenization with MeCab morphological parser."""
def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, mecab_option=None): def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, mecab_option: Optional[str] = None):
"""Constructs a MecabTokenizer. """Constructs a MecabTokenizer.
Args: Args:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment