custom_tokenization_fast.py 193 Bytes
Newer Older
1
2
3
4
5
6
7
8
from transformers import BertTokenizerFast

from .custom_tokenization import CustomTokenizer


class CustomTokenizerFast(BertTokenizerFast):
    slow_tokenizer_class = CustomTokenizer
    pass