tokenization_utils.py 48.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# coding=utf-8
# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for OpenAI GPT."""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import logging
import os
21
22
import json
import six
23
import copy
24
25
from io import open

thomwolf's avatar
thomwolf committed
26
from .file_utils import cached_path, is_tf_available, is_torch_available
thomwolf's avatar
thomwolf committed
27
28
29

if is_tf_available():
    import tensorflow as tf
30
31
32

logger = logging.getLogger(__name__)

33
34
SPECIAL_TOKENS_MAP_FILE = 'special_tokens_map.json'
ADDED_TOKENS_FILE = 'added_tokens.json'
35
TOKENIZER_CONFIG_FILE = 'tokenizer_config.json'
36
37

class PreTrainedTokenizer(object):
38
39
    """ Base class for all tokenizers.
    Handle all the shared methods for tokenization and special tokens as well as methods dowloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary.
40

41
    This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
42

43
44
45
46
47
    Class attributes (overridden by derived classes):

        - ``vocab_files_names``: a python ``dict`` with, as keys, the ``__init__`` keyword name of each vocabulary file required by the model, and as associated values, the filename for saving the associated file (string).
        - ``pretrained_vocab_files_map``: a python ``dict of dict`` the high-level keys being the ``__init__`` keyword name of each vocabulary file required by the model, the low-level being the `short-cut-names` (string) of the pretrained models with, as associated values, the `url` (string) to the associated pretrained vocabulary file.
        - ``max_model_input_sizes``: a python ``dict`` with, as keys, the `short-cut-names` (string) of the pretrained models, and as associated values, the maximum length of the sequence inputs of this model, or None if the model has no maximum input size.
48
        - ``pretrained_init_configuration``: a python ``dict`` with, as keys, the `short-cut-names` (string) of the pretrained models, and as associated values, a dictionnary of specific arguments to pass to the ``__init__``method of the tokenizer class for this pretrained model when loading the tokenizer with the ``from_pretrained()`` method.
49
50
51

    Parameters:

thomwolf's avatar
thomwolf committed
52
        - ``bos_token``: (`Optional`) string: a beginning of sentence token. Will be associated to ``self.bos_token`` and ``self.bos_token_id``
53

thomwolf's avatar
thomwolf committed
54
        - ``eos_token``: (`Optional`) string: an end of sentence token. Will be associated to ``self.eos_token`` and ``self.eos_token_id``
55

thomwolf's avatar
thomwolf committed
56
        - ``unk_token``: (`Optional`) string: an unknown token. Will be associated to ``self.unk_token`` and ``self.unk_token_id``
57

thomwolf's avatar
thomwolf committed
58
        - ``sep_token``: (`Optional`) string: a separation token (e.g. to separate context and query in an input sequence). Will be associated to ``self.sep_token`` and ``self.sep_token_id``
59

thomwolf's avatar
thomwolf committed
60
        - ``pad_token``: (`Optional`) string: a padding token. Will be associated to ``self.pad_token`` and ``self.pad_token_id``
61

thomwolf's avatar
thomwolf committed
62
        - ``cls_token``: (`Optional`) string: a classification token (e.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model). Will be associated to ``self.cls_token`` and ``self.cls_token_id``
63

thomwolf's avatar
thomwolf committed
64
        - ``mask_token``: (`Optional`) string: a masking token (e.g. when training a model with masked-language modeling). Will be associated to ``self.mask_token`` and ``self.mask_token_id``
65

thomwolf's avatar
thomwolf committed
66
        - ``additional_special_tokens``: (`Optional`) list: a list of additional special tokens. Adding all special tokens here ensure they won't be split by the tokenization process. Will be associated to ``self.additional_special_tokens`` and ``self.additional_special_tokens_ids``
67
68
69
    """
    vocab_files_names = {}
    pretrained_vocab_files_map = {}
70
    pretrained_init_configuration = {}
71
72
    max_model_input_sizes = {}

73
74
75
76
77
78
    SPECIAL_TOKENS_ATTRIBUTES = ["bos_token", "eos_token", "unk_token", "sep_token",
                                 "pad_token", "cls_token", "mask_token",
                                 "additional_special_tokens"]

    @property
    def bos_token(self):
79
        """ Beginning of sentence token (string). Log an error if used while not having been set. """
80
81
82
83
84
85
        if self._bos_token is None:
            logger.error("Using bos_token, but it is not set yet.")
        return self._bos_token

    @property
    def eos_token(self):
86
        """ End of sentence token (string). Log an error if used while not having been set. """
87
88
89
90
91
92
        if self._eos_token is None:
            logger.error("Using eos_token, but it is not set yet.")
        return self._eos_token

    @property
    def unk_token(self):
93
        """ Unknown token (string). Log an error if used while not having been set. """
94
95
96
97
98
99
        if self._unk_token is None:
            logger.error("Using unk_token, but it is not set yet.")
        return self._unk_token

    @property
    def sep_token(self):
100
        """ Separation token (string). E.g. separate context and query in an input sequence. Log an error if used while not having been set. """
101
102
103
104
105
106
        if self._sep_token is None:
            logger.error("Using sep_token, but it is not set yet.")
        return self._sep_token

    @property
    def pad_token(self):
107
        """ Padding token (string). Log an error if used while not having been set. """
108
109
110
111
112
113
        if self._pad_token is None:
            logger.error("Using pad_token, but it is not set yet.")
        return self._pad_token

    @property
    def cls_token(self):
114
        """ Classification token (string). E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """
115
116
117
118
119
120
        if self._cls_token is None:
            logger.error("Using cls_token, but it is not set yet.")
        return self._cls_token

    @property
    def mask_token(self):
121
        """ Mask token (string). E.g. when training a model with masked-language modeling. Log an error if used while not having been set. """
122
123
124
125
126
127
        if self._mask_token is None:
            logger.error("Using mask_token, but it is not set yet.")
        return self._mask_token

    @property
    def additional_special_tokens(self):
128
        """ All the additional special tokens you may want to use (list of strings). Log an error if used while not having been set. """
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        if self._additional_special_tokens is None:
            logger.error("Using additional_special_tokens, but it is not set yet.")
        return self._additional_special_tokens

    @bos_token.setter
    def bos_token(self, value):
        self._bos_token = value

    @eos_token.setter
    def eos_token(self, value):
        self._eos_token = value

    @unk_token.setter
    def unk_token(self, value):
        self._unk_token = value

    @sep_token.setter
    def sep_token(self, value):
        self._sep_token = value

    @pad_token.setter
    def pad_token(self, value):
        self._pad_token = value

    @cls_token.setter
    def cls_token(self, value):
        self._cls_token = value

    @mask_token.setter
    def mask_token(self, value):
        self._mask_token = value

    @additional_special_tokens.setter
    def additional_special_tokens(self, value):
        self._additional_special_tokens = value

165
166
167
    @property
    def bos_token_id(self):
        """ Id of the beginning of sentence token in the vocabulary. Log an error if used while not having been set. """
168
        return self.convert_tokens_to_ids(self.bos_token)
169
170
171
172

    @property
    def eos_token_id(self):
        """ Id of the end of sentence token in the vocabulary. Log an error if used while not having been set. """
173
        return self.convert_tokens_to_ids(self.eos_token)
174
175

    @property
maru0kun's avatar
maru0kun committed
176
    def unk_token_id(self):
177
        """ Id of the unknown token in the vocabulary. Log an error if used while not having been set. """
178
        return self.convert_tokens_to_ids(self.unk_token)
179
180
181
182

    @property
    def sep_token_id(self):
        """ Id of the separation token in the vocabulary. E.g. separate context and query in an input sequence. Log an error if used while not having been set. """
183
        return self.convert_tokens_to_ids(self.sep_token)
184
185
186
187

    @property
    def pad_token_id(self):
        """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """
188
        return self.convert_tokens_to_ids(self.pad_token)
189
190
191
192

    @property
    def cls_token_id(self):
        """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """
193
        return self.convert_tokens_to_ids(self.cls_token)
194
195
196
197

    @property
    def mask_token_id(self):
        """ Id of the mask token in the vocabulary. E.g. when training a model with masked-language modeling. Log an error if used while not having been set. """
198
        return self.convert_tokens_to_ids(self.mask_token)
199
200
201
202

    @property
    def additional_special_tokens_ids(self):
        """ Ids of all the additional special tokens in the vocabulary (list of integers). Log an error if used while not having been set. """
203
        return self.convert_tokens_to_ids(self.additional_special_tokens)
204

205
206
207
208
209
210
211
212
213
214
215
    def __init__(self, max_len=None, **kwargs):
        self._bos_token = None
        self._eos_token = None
        self._unk_token = None
        self._sep_token = None
        self._pad_token = None
        self._cls_token = None
        self._mask_token = None
        self._additional_special_tokens = []

        self.max_len = max_len if max_len is not None else int(1e12)
216
217

        # Added tokens
218
219
220
        self.added_tokens_encoder = {}
        self.added_tokens_decoder = {}

221
222
223
224
        # inputs and kwargs for saving and re-loading (see ``from_pretrained`` and ``save_pretrained``)
        self.init_inputs = ()
        self.init_kwargs = {}

225
        for key, value in kwargs.items():
226
            if key in self.SPECIAL_TOKENS_ATTRIBUTES:
227
228
229
230
                if key == 'additional_special_tokens':
                    assert isinstance(value, (list, tuple)) and all(isinstance(t, str) or (six.PY2 and isinstance(t, unicode)) for t in value)
                else:
                    assert isinstance(value, str) or (six.PY2 and isinstance(value, unicode))
231
232
233
                setattr(self, key, value)


234
235
    @classmethod
    def from_pretrained(cls, *inputs, **kwargs):
LysandreJik's avatar
Doc  
LysandreJik committed
236
237
        r"""
        Instantiate a :class:`~pytorch_transformers.PreTrainedTokenizer` (or a derived class) from a predefined tokenizer.
238

LysandreJik's avatar
Doc  
LysandreJik committed
239
        Args:
240
241
242
243
244
245
246
247
248
            pretrained_model_name_or_path: either:

                - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``.
                - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~pytorch_transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``.
                - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``.

            cache_dir: (`optional`) string:
                Path to a directory in which a downloaded predefined tokenizer vocabulary files should be cached if the standard cache should not be used.

249
250
251
            force_download: (`optional`) boolean, default False:
                Force to (re-)download the vocabulary files and override the cached versions if they exists.

252
253
254
255
            proxies: (`optional`) dict, default None:
                A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
                The proxies are used on each request.

256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
            inputs: (`optional`) positional arguments: will be passed to the Tokenizer ``__init__`` method.

            kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~pytorch_transformers.PreTrainedTokenizer` for details.

        Examples::

            # We can't instantiate directly the base class `PreTrainedTokenizer` so let's show our examples on a derived class: BertTokenizer

            # Download vocabulary from S3 and cache.
            tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

            # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`)
            tokenizer = BertTokenizer.from_pretrained('./test/saved_model/')

            # If the tokenizer uses a single vocabulary file, you can point directly to this file
            tokenizer = BertTokenizer.from_pretrained('./test/saved_model/my_vocab.txt')

            # You can link tokens to special vocabulary when instantiating
            tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', unk_token='<unk>')
            # You should be sure '<unk>' is in the vocabulary when doing that.
            # Otherwise use tokenizer.add_special_tokens({'unk_token': '<unk>'}) instead)
            assert tokenizer.unk_token == '<unk>'

        """
280
281
        return cls._from_pretrained(*inputs, **kwargs)

282

283
    @classmethod
284
    def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs):
thomwolf's avatar
thomwolf committed
285
        cache_dir = kwargs.pop('cache_dir', None)
286
        force_download = kwargs.pop('force_download', False)
287
        proxies = kwargs.pop('proxies', None)
thomwolf's avatar
thomwolf committed
288

289
290
        s3_models = list(cls.max_model_input_sizes.keys())
        vocab_files = {}
291
        init_configuration = {}
292
        if pretrained_model_name_or_path in s3_models:
thomwolf's avatar
thomwolf committed
293
            # Get the vocabulary from AWS S3 bucket
294
295
            for file_id, map_list in cls.pretrained_vocab_files_map.items():
                vocab_files[file_id] = map_list[pretrained_model_name_or_path]
296
297
            if cls.pretrained_init_configuration and pretrained_model_name_or_path in cls.pretrained_init_configuration:
                init_configuration = cls.pretrained_init_configuration[pretrained_model_name_or_path]
298
        else:
thomwolf's avatar
thomwolf committed
299
            # Get the vocabulary from local files
300
301
302
303
304
            logger.info(
                "Model name '{}' not found in model shortcut name list ({}). "
                "Assuming '{}' is a path or url to a directory containing tokenizer files.".format(
                    pretrained_model_name_or_path, ', '.join(s3_models),
                    pretrained_model_name_or_path))
thomwolf's avatar
thomwolf committed
305
306
307

            # Look for the tokenizer main vocabulary files
            for file_id, file_name in cls.vocab_files_names.items():
308
                if os.path.isdir(pretrained_model_name_or_path):
thomwolf's avatar
thomwolf committed
309
                    # If a directory is provided we look for the standard filenames
310
311
                    full_file_name = os.path.join(pretrained_model_name_or_path, file_name)
                else:
thomwolf's avatar
thomwolf committed
312
                    # If a path to a file is provided we use it (will only work for non-BPE tokenizer using a single vocabulary file)
313
314
                    full_file_name = pretrained_model_name_or_path
                if not os.path.exists(full_file_name):
315
                    logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
316
317
                    full_file_name = None
                vocab_files[file_id] = full_file_name
thomwolf's avatar
thomwolf committed
318
319

            # Look for the additional tokens files
320
321
322
323
            additional_files_names = {'added_tokens_file': ADDED_TOKENS_FILE,
                                      'special_tokens_map_file': SPECIAL_TOKENS_MAP_FILE,
                                      'tokenizer_config_file': TOKENIZER_CONFIG_FILE,
                                      }
thomwolf's avatar
thomwolf committed
324
325
326
327
328
329

            # If a path to a file was provided, get the parent directory
            saved_directory = pretrained_model_name_or_path
            if os.path.exists(saved_directory) and not os.path.isdir(saved_directory):
                saved_directory = os.path.dirname(saved_directory)

330
            for file_id, file_name in additional_files_names.items():
thomwolf's avatar
thomwolf committed
331
332
333
334
335
336
                full_file_name = os.path.join(saved_directory, file_name)
                if not os.path.exists(full_file_name):
                    logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
                    full_file_name = None
                vocab_files[file_id] = full_file_name

337
338
339
340
341
342
343
344
            if all(full_file_name is None for full_file_name in vocab_files.values()):
                logger.error(
                    "Model name '{}' was not found in model name list ({}). "
                    "We assumed '{}' was a path or url but couldn't find tokenizer files"
                    "at this path or url.".format(
                        pretrained_model_name_or_path, ', '.join(s3_models),
                        pretrained_model_name_or_path, ))
                return None
345
346

        # Get files from url, cache, or disk depending on the case
347
348
349
350
351
352
        try:
            resolved_vocab_files = {}
            for file_id, file_path in vocab_files.items():
                if file_path is None:
                    resolved_vocab_files[file_id] = None
                else:
353
                    resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies)
354
        except EnvironmentError as e:
355
356
357
358
359
360
361
362
363
            if pretrained_model_name_or_path in s3_models:
                logger.error("Couldn't reach server to download vocabulary.")
            else:
                logger.error(
                    "Model name '{}' was not found in model name list ({}). "
                    "We assumed '{}' was a path or url but couldn't find files {} "
                    "at this path or url.".format(
                        pretrained_model_name_or_path, ', '.join(s3_models),
                        pretrained_model_name_or_path, str(vocab_files.keys())))
364
            raise e
365
366
367
368
369
370
371
372

        for file_id, file_path in vocab_files.items():
            if file_path == resolved_vocab_files[file_id]:
                logger.info("loading file {}".format(file_path))
            else:
                logger.info("loading file {} from cache at {}".format(
                    file_path, resolved_vocab_files[file_id]))

373
374
375
376
377
        # Prepare tokenizer initialization kwargs
        # Did we saved some inputs and kwargs to reload ?
        tokenizer_config_file = resolved_vocab_files.pop('tokenizer_config_file', None)
        if tokenizer_config_file is not None:
            init_kwargs = json.load(open(tokenizer_config_file, encoding="utf-8"))
378
            saved_init_inputs = init_kwargs.pop('init_inputs', ())
379
380
381
382
383
384
            if not init_inputs:
                init_inputs = saved_init_inputs
        else:
            init_kwargs = init_configuration

        # Update with newly provided kwargs
385
386
        init_kwargs.update(kwargs)

387
        # Set max length if needed
388
389
390
391
        if pretrained_model_name_or_path in cls.max_model_input_sizes:
            # if we're using a pretrained model, ensure the tokenizer
            # wont index sequences longer than the number of positional embeddings
            max_len = cls.max_model_input_sizes[pretrained_model_name_or_path]
392
            if max_len is not None and isinstance(max_len, (int, float)):
393
                init_kwargs['max_len'] = min(init_kwargs.get('max_len', int(1e12)), max_len)
394

395
        # Merge resolved_vocab_files arguments in init_kwargs.
396
397
        added_tokens_file = resolved_vocab_files.pop('added_tokens_file', None)
        special_tokens_map_file = resolved_vocab_files.pop('special_tokens_map_file', None)
thomwolf's avatar
thomwolf committed
398
        for args_name, file_path in resolved_vocab_files.items():
399
400
            if args_name not in init_kwargs:
                init_kwargs[args_name] = file_path
401
402
403
        if special_tokens_map_file is not None:
            special_tokens_map = json.load(open(special_tokens_map_file, encoding="utf-8"))
            for key, value in special_tokens_map.items():
404
405
                if key not in init_kwargs:
                    init_kwargs[key] = value
thomwolf's avatar
thomwolf committed
406

407
        # Instantiate tokenizer.
408
409
410
411
412
        tokenizer = cls(*init_inputs, **init_kwargs)

        # Save inputs and kwargs for saving and re-loading with ``save_pretrained``
        tokenizer.init_inputs = init_inputs
        tokenizer.init_kwargs = init_kwargs
413

414
415
        # Add supplementary tokens.
        if added_tokens_file is not None:
thomwolf's avatar
thomwolf committed
416
            added_tok_encoder = json.load(open(added_tokens_file, encoding="utf-8"))
417
418
419
420
            added_tok_decoder = {v:k for k, v in added_tok_encoder.items()}
            tokenizer.added_tokens_encoder.update(added_tok_encoder)
            tokenizer.added_tokens_decoder.update(added_tok_decoder)

421
422
        return tokenizer

thomwolf's avatar
thomwolf committed
423

424
    def save_pretrained(self, save_directory):
425
426
427
428
429
430
431
        """ Save the tokenizer vocabulary files together with:
                - added tokens,
                - special-tokens-to-class-attributes-mapping,
                - tokenizer instantiation positional and keywords inputs (e.g. do_lower_case for Bert).

            This won't save modifications other than (added tokens and special token mapping) you may have
            applied to the tokenizer after the instantion (e.g. modifying tokenizer.do_lower_case after creation).
432
433

            This method make sure the full tokenizer can then be re-loaded using the :func:`~pytorch_transformers.PreTrainedTokenizer.from_pretrained` class method.
434
435
436
437
438
439
440
        """
        if not os.path.isdir(save_directory):
            logger.error("Saving directory ({}) should be a directory".format(save_directory))
            return

        special_tokens_map_file = os.path.join(save_directory, SPECIAL_TOKENS_MAP_FILE)
        added_tokens_file = os.path.join(save_directory, ADDED_TOKENS_FILE)
441
442
443
444
        tokenizer_config_file = os.path.join(save_directory, TOKENIZER_CONFIG_FILE)

        tokenizer_config = copy.deepcopy(self.init_kwargs)
        tokenizer_config['init_inputs'] = copy.deepcopy(self.init_inputs)
445
446
        for file_id in self.vocab_files_names.keys():
            tokenizer_config.pop(file_id, None)
447
448
449

        with open(tokenizer_config_file, 'w', encoding='utf-8') as f:
            f.write(json.dumps(tokenizer_config, ensure_ascii=False))
450
451
452
453
454

        with open(special_tokens_map_file, 'w', encoding='utf-8') as f:
            f.write(json.dumps(self.special_tokens_map, ensure_ascii=False))

        with open(added_tokens_file, 'w', encoding='utf-8') as f:
thomwolf's avatar
thomwolf committed
455
            if self.added_tokens_encoder:
456
                out_str = json.dumps(self.added_tokens_encoder, ensure_ascii=False)
thomwolf's avatar
thomwolf committed
457
458
459
            else:
                out_str = u"{}"
            f.write(out_str)
460
461
462
463
464
465
466

        vocab_files = self.save_vocabulary(save_directory)

        return vocab_files + (special_tokens_map_file, added_tokens_file)


    def save_vocabulary(self, save_directory):
467
        """ Save the tokenizer vocabulary to a directory. This method does *NOT* save added tokens
468
            and special token mappings.
469
470

            Please use :func:`~pytorch_transformers.PreTrainedTokenizer.save_pretrained` `()` to save the full Tokenizer state if you want to reload it using the :func:`~pytorch_transformers.PreTrainedTokenizer.from_pretrained` class method.
471
        """
thomwolf's avatar
thomwolf committed
472
473
        raise NotImplementedError

474
475

    def vocab_size(self):
476
        """ Size of the base vocabulary (without the added tokens) """
thomwolf's avatar
thomwolf committed
477
478
        raise NotImplementedError

479
480

    def __len__(self):
481
        """ Size of the full vocabulary with the added tokens """
482
483
484
485
        return self.vocab_size + len(self.added_tokens_encoder)


    def add_tokens(self, new_tokens):
LysandreJik's avatar
Doc  
LysandreJik committed
486
487
        """
        Add a list of new tokens to the tokenizer class. If the new tokens are not in the
488
489
        vocabulary, they are added to it with indices starting from length of the current vocabulary.

LysandreJik's avatar
Doc  
LysandreJik committed
490
491
        Args:
            new_tokens: list of string. Each string is a token to add. Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer assign the index of the ``unk_token`` to them).
492

LysandreJik's avatar
Doc  
LysandreJik committed
493
494
        Returns:
            Number of tokens added to the vocabulary.
495
496
497
498
499
500
501
502
503
504

        Examples::

            # Let's see how to increase the vocabulary of Bert model and tokenizer
            tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
            model = BertModel.from_pretrained('bert-base-uncased')

            num_added_toks = tokenizer.add_tokens(['new_tok1', 'my_new-tok2'])
            print('We have added', num_added_toks, 'tokens')
            model.resize_token_embeddings(len(tokenizer))  # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e. the length of the tokenizer.
505
506
507
508
509
510
        """
        if not new_tokens:
            return 0

        to_add_tokens = []
        for token in new_tokens:
511
            assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode))
thomwolf's avatar
thomwolf committed
512
513
            if token != self.unk_token and \
                    self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token):
514
515
516
517
518
519
520
521
522
523
                to_add_tokens.append(token)
                logger.info("Adding %s to the vocabulary", token)

        added_tok_encoder = dict((tok, len(self) + i) for i, tok in enumerate(to_add_tokens))
        added_tok_decoder = {v:k for k, v in added_tok_encoder.items()}
        self.added_tokens_encoder.update(added_tok_encoder)
        self.added_tokens_decoder.update(added_tok_decoder)

        return len(to_add_tokens)

524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
    def num_added_tokens(self, pair=False):
        """
        Returns the number of added tokens when encoding a sequence with special tokens.

        Note:
            This encodes inputs and checks the number of added tokens, and is therefore not efficient. Do not put this
            inside your training loop.

        Args:
            pair: Returns the number of added tokens in the case of a sequence pair if set to True, returns the
                number of added tokens in the case of a single sequence if set to False.

        Returns:
            Number of tokens added to sequences
        """

        if pair:
541
            initial_tokens_len = len(self.encode("This is a sequence") + self.encode("This is another"))
LysandreJik's avatar
LysandreJik committed
542
            final_tokens_len = len(self.encode("This is a sequence", "This is another", add_special_tokens=True))
543
544
545
546
547
        else:
            initial_tokens_len = len(self.encode("This is a sequence"))
            final_tokens_len = len(self.encode("This is a sequence", add_special_tokens=True))

        return final_tokens_len - initial_tokens_len
548
549

    def add_special_tokens(self, special_tokens_dict):
LysandreJik's avatar
Doc  
LysandreJik committed
550
551
552
553
        """
        Add a dictionary of special tokens (eos, pad, cls...) to the encoder and link them
        to class attributes. If special tokens are NOT in the vocabulary, they are added
        to it (indexed starting from the last index of the current vocabulary).
554

thomwolf's avatar
thomwolf committed
555
556
557
558
559
560
561
        Using `add_special_tokens` will ensure your special tokens can be used in several ways:

        - special tokens are carefully handled by the tokenizer (they are never split)
        - you can easily refer to special tokens using tokenizer class attributes like `tokenizer.cls_token`. This makes it easy to develop model-agnostic training and fine-tuning scripts.

        When possible, special tokens are already registered for provided pretrained models (ex: BertTokenizer cls_token is already registered to be '[CLS]' and XLM's one is also registered to be '</s>')

LysandreJik's avatar
Doc  
LysandreJik committed
562
563
564
565
        Args:
            special_tokens_dict: dict of string. Keys should be in the list of predefined special attributes:
                [``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``,
                ``additional_special_tokens``].
566

LysandreJik's avatar
Doc  
LysandreJik committed
567
                Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer assign the index of the ``unk_token`` to them).
568

LysandreJik's avatar
Doc  
LysandreJik committed
569
570
        Returns:
            Number of tokens added to the vocabulary.
571
572
573
574
575
576
577
578
579
580
581
582
583
584

        Examples::

            # Let's see how to add a new classification token to GPT-2
            tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
            model = GPT2Model.from_pretrained('gpt2')

            special_tokens_dict = {'cls_token': '<CLS>'}

            num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)
            print('We have added', num_added_toks, 'tokens')
            model.resize_token_embeddings(len(tokenizer))  # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e. the length of the tokenizer.

            assert tokenizer.cls_token == '<CLS>'
585
586
587
588
        """
        if not special_tokens_dict:
            return 0

589
        added_tokens = 0
590
        for key, value in special_tokens_dict.items():
591
            assert key in self.SPECIAL_TOKENS_ATTRIBUTES
592
593
594
595
596
597
            if key == 'additional_special_tokens':
                assert isinstance(value, (list, tuple)) and all(isinstance(t, str) or (six.PY2 and isinstance(t, unicode)) for t in value)
                added_tokens += self.add_tokens(value)
            else:
                assert isinstance(value, str) or (six.PY2 and isinstance(value, unicode))
                added_tokens += self.add_tokens([value])
598
599
600
            logger.info("Assigning %s to the %s key of the tokenizer", value, key)
            setattr(self, key, value)

601
        return added_tokens
602
603
604
605
606
607
608
609

    def tokenize(self, text, **kwargs):
        """ Converts a string in a sequence of tokens (string), using the tokenizer.
            Split in words for word-based vocabulary or sub-words for sub-word-based
            vocabularies (BPE/SentencePieces/WordPieces).

            Take care of added tokens.
        """
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
        def split_on_token(tok, text):
            result = []
            split_text = text.split(tok)
            for i, sub_text in enumerate(split_text):
                sub_text = sub_text.strip()
                if i == 0 and not sub_text:
                    result += [tok]
                elif i == len(split_text) - 1:
                    if sub_text:
                        result += [sub_text]
                    else:
                        pass
                else:
                    if sub_text:
                        result += [sub_text]
                    result += [tok]
            return result

628
629
630
631
632
        def split_on_tokens(tok_list, text):
            if not text:
                return []
            if not tok_list:
                return self._tokenize(text, **kwargs)
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648

            tokenized_text = []
            text_list = [text]
            for tok in tok_list:
                tokenized_text = []
                for sub_text in text_list:
                    if sub_text not in self.added_tokens_encoder \
                            and sub_text not in self.all_special_tokens:
                        tokenized_text += split_on_token(tok, sub_text)
                    else:
                        tokenized_text += [sub_text]
                text_list = tokenized_text

            return sum((self._tokenize(token, **kwargs) if token not \
                    in self.added_tokens_encoder and token not in self.all_special_tokens \
                    else [token] for token in tokenized_text), [])
649

650
        added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens
651
652
653
654
655
656
657
658
        tokenized_text = split_on_tokens(added_tokens, text)
        return tokenized_text

    def _tokenize(self, text, **kwargs):
        """ Converts a string in a sequence of tokens (string), using the tokenizer.
            Split in words for word-based vocabulary or sub-words for sub-word-based
            vocabularies (BPE/SentencePieces/WordPieces).

659
            Do NOT take care of added tokens.
660
        """
thomwolf's avatar
thomwolf committed
661
662
        raise NotImplementedError

663
    def convert_tokens_to_ids(self, tokens):
664
665
        """ Converts a single token, or a sequence of tokens, (str/unicode) in a single integer id
            (resp. a sequence of ids), using the vocabulary.
666
        """
667
668
669
        if tokens is None:
            return None

670
        if isinstance(tokens, str) or (six.PY2 and isinstance(tokens, unicode)):
671
            return self._convert_token_to_id_with_added_voc(tokens)
672
673
674

        ids = []
        for token in tokens:
675
            ids.append(self._convert_token_to_id_with_added_voc(token))
676
677
678
679
680
681
        if len(ids) > self.max_len:
            logger.warning("Token indices sequence length is longer than the specified maximum sequence length "
                           "for this model ({} > {}). Running this sequence through the model will result in "
                           "indexing errors".format(len(ids), self.max_len))
        return ids

682
    def _convert_token_to_id_with_added_voc(self, token):
683
684
685
        if token is None:
            return None

686
687
688
689
690
        if token in self.added_tokens_encoder:
            return self.added_tokens_encoder[token]
        return self._convert_token_to_id(token)

    def _convert_token_to_id(self, token):
thomwolf's avatar
thomwolf committed
691
692
        raise NotImplementedError

thomwolf's avatar
thomwolf committed
693
694
695
696
697
698
699
700
701
    def encode(self,
                text,
                text_pair=None,
                add_special_tokens=False,
                max_length=None,
                stride=0,
                truncate_first_sequence=True,
                return_tensors=None,
                **kwargs):
LysandreJik's avatar
Doc  
LysandreJik committed
702
703
        """
        Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary.
704

LysandreJik's avatar
Doc  
LysandreJik committed
705
706
707
        Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``.

        Args:
LysandreJik's avatar
LysandreJik committed
708
709
710
711
712
713
            text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using
                the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
                method)
            text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized
                string using the `tokenize` method) or a list of integers (tokenized string ids using the
                `convert_tokens_to_ids` method)
LysandreJik's avatar
Doc  
LysandreJik committed
714
715
            add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative
                to their model.
thomwolf's avatar
thomwolf committed
716
717
718
719
720
721
722
723
            max_length: if set to a number, will limit the total sequence returned so that it has a maximum length.
                If there are overflowing tokens, those will be added to the returned dictionary
            stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens
                from the main sequence returned. The value of this argument defined the number of additional tokens.
            truncate_first_sequence: if there is a specified max_length, this flag will choose which sequence
                will be truncated.
            return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant
                or PyTorch torch.Tensor instead of a list of python integers.
thomwolf's avatar
thomwolf committed
724
            **kwargs: passed to the `self.tokenize()` method
725
        """
thomwolf's avatar
thomwolf committed
726
727
728
729
730
731
732
733
        encoded_inputs = self.encode_plus(text,
                                          text_pair=text_pair,
                                          max_length=max_length,
                                          add_special_tokens=add_special_tokens,
                                          stride=stride,
                                          truncate_first_sequence=truncate_first_sequence,
                                          return_tensors=return_tensors,
                                          **kwargs)
thomwolf's avatar
thomwolf committed
734
735

        return encoded_inputs["input_ids"]
736

737
738
739
740
741
742
    def encode_plus(self,
                    text,
                    text_pair=None,
                    add_special_tokens=False,
                    max_length=None,
                    stride=0,
LysandreJik's avatar
LysandreJik committed
743
                    truncate_first_sequence=True,
thomwolf's avatar
thomwolf committed
744
                    return_tensors=None,
745
                    **kwargs):
746
        """
thomwolf's avatar
thomwolf committed
747
748
        Returns a dictionary containing the encoded sequence or sequence pair and additional informations:
        the mask for sequence classification and the overflowing elements if a ``max_length`` is specified.
LysandreJik's avatar
Doc  
LysandreJik committed
749
750

        Args:
LysandreJik's avatar
LysandreJik committed
751
752
753
754
755
756
            text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using
                the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
                method)
            text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized
                string using the `tokenize` method) or a list of integers (tokenized string ids using the
                `convert_tokens_to_ids` method)
LysandreJik's avatar
Doc  
LysandreJik committed
757
758
            add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative
                to their model.
759
            max_length: if set to a number, will limit the total sequence returned so that it has a maximum length.
LysandreJik's avatar
LysandreJik committed
760
761
762
                If there are overflowing tokens, those will be added to the returned dictionary
            stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens
                from the main sequence returned. The value of this argument defined the number of additional tokens.
LysandreJik's avatar
LysandreJik committed
763
            truncate_first_sequence: if there is a specified max_length, this flag will choose which sequence
764
                will be truncated.
thomwolf's avatar
thomwolf committed
765
766
            return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant
                or PyTorch torch.Tensor instead of a list of python integers.
thomwolf's avatar
thomwolf committed
767
            **kwargs: passed to the `self.tokenize()` method
768
        """
769

LysandreJik's avatar
LysandreJik committed
770
771
        def get_input_ids(text):
            if isinstance(text, six.string_types):
thomwolf's avatar
thomwolf committed
772
                return self.convert_tokens_to_ids(self.tokenize(text, **kwargs))
LysandreJik's avatar
LysandreJik committed
773
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], six.string_types):
thomwolf's avatar
thomwolf committed
774
                return self.convert_tokens_to_ids(text)
LysandreJik's avatar
LysandreJik committed
775
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
thomwolf's avatar
thomwolf committed
776
                return text
777
            else:
LysandreJik's avatar
LysandreJik committed
778
779
                raise ValueError("Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.")

thomwolf's avatar
thomwolf committed
780
781
        first_ids = get_input_ids(text)
        second_ids = get_input_ids(text_pair) if text_pair is not None else None
782

thomwolf's avatar
thomwolf committed
783
784
785
786
787
        return self.prepare_for_model(first_ids,
                                      pair_ids=second_ids,
                                      max_length=max_length,
                                      add_special_tokens=add_special_tokens,
                                      stride=stride,
thomwolf's avatar
thomwolf committed
788
789
                                      truncate_first_sequence=truncate_first_sequence,
                                      return_tensors=return_tensors)
790
791


thomwolf's avatar
thomwolf committed
792
793
    def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0,
                          truncate_first_sequence=True, return_tensors=None):
LysandreJik's avatar
LysandreJik committed
794
        """
thomwolf's avatar
thomwolf committed
795
796
        Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model.
        It adds special tokens, truncates
LysandreJik's avatar
LysandreJik committed
797
798
799
800
801
802
        sequences if overflowing while taking into account the special tokens and manages a window stride for
        overflowing tokens

        Args:
            ids: list of tokenized input ids. Can be obtained from a string by chaining the
                `tokenize` and `convert_tokens_to_ids` methods.
thomwolf's avatar
thomwolf committed
803
            pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the
LysandreJik's avatar
LysandreJik committed
804
805
                `tokenize` and `convert_tokens_to_ids` methods.
            max_length: maximum length of the returned list. Will truncate by taking into account the special tokens.
thomwolf's avatar
thomwolf committed
806
807
            add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative
                to their model.
LysandreJik's avatar
LysandreJik committed
808
809
            stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential
                list of inputs.
thomwolf's avatar
thomwolf committed
810
811
812
            truncate_first_sequence: if set to `True` and an optional second list of input ids is provided,
                alongside a specified `max_length`, will truncate the first sequence if the total size is superior
                than the specified `max_length`. If set to `False`, will truncate the second sequence instead.
thomwolf's avatar
thomwolf committed
813
814
            return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant
                or PyTorch torch.Tensor instead of a list of python integers.
LysandreJik's avatar
LysandreJik committed
815
816
817
818

        Return:
            a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given.
        """
thomwolf's avatar
thomwolf committed
819
820
821
        pair = bool(pair_ids is not None)
        len_ids = len(ids)
        len_pair_ids = len(pair_ids) if pair else 0
822

thomwolf's avatar
thomwolf committed
823
        encoded_inputs = {}
824
        if max_length:
thomwolf's avatar
thomwolf committed
825
826
            n_added_tokens = self.num_added_tokens(pair=pair) if add_special_tokens else 0
            if pair and n_added_tokens + (len_pair_ids if truncate_first_sequence else len_ids) >= max_length:
827
                logger.warning(
thomwolf's avatar
thomwolf committed
828
829
                    "You supplied a pair of sequence in which the sequence that will not be truncated is longer than the maximum specified length."
                    "This pair of sequences will not be truncated.")
thomwolf's avatar
thomwolf committed
830
            else:
thomwolf's avatar
thomwolf committed
831
832
833
834
835
836
837
                if n_added_tokens + len_ids + len_pair_ids > max_length:
                    if truncate_first_sequence or not pair:
                        encoded_inputs["overflowing_tokens"] = ids[max_length - len_pair_ids - n_added_tokens - stride:]
                        ids = ids[:max_length - len_pair_ids - n_added_tokens]
                    elif not truncate_first_sequence and pair:
                        encoded_inputs["overflowing_tokens"] = pair_ids[max_length - len_ids - n_added_tokens - stride:]
                        pair_ids = pair_ids[:max_length - len_ids - n_added_tokens]
LysandreJik's avatar
LysandreJik committed
838
                    else:
thomwolf's avatar
thomwolf committed
839
840
841
842
843
844
845
846
847
                        logger.warning(
                            "Cannot truncate second sequence as it is not provided. No truncation.")

        if add_special_tokens:
            sequence = self.add_special_tokens_sequence_pair(ids, pair_ids) if pair else self.add_special_tokens_single_sequence(ids)
            token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) if pair else [0] * len(sequence)
        else:
            sequence = ids + pair_ids if pair else ids
            token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else [])
848

thomwolf's avatar
thomwolf committed
849
850
851
852
853
        if return_tensors == 'tf' and is_tf_available():
            sequence = tf.constant(sequence)
            token_type_ids = tf.constant(token_type_ids)
        elif return_tensors = 'pt' and is

thomwolf's avatar
thomwolf committed
854
855
        encoded_inputs["input_ids"] = sequence
        encoded_inputs["token_type_ids"] = token_type_ids
856

thomwolf's avatar
thomwolf committed
857
        return encoded_inputs
thomwolf's avatar
thomwolf committed
858

thomwolf's avatar
thomwolf committed
859
    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1):
860
        logger.warning("This tokenizer does not make use of special tokens.")
thomwolf's avatar
thomwolf committed
861
        return [0] * len(token_ids_0) + [1] * len(token_ids_1)
862

863
    def add_special_tokens_single_sequence(self, token_ids):
LysandreJik's avatar
LysandreJik committed
864
865
        logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.")
        return token_ids
866

867
    def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1):
LysandreJik's avatar
LysandreJik committed
868
869
        logger.warning("This tokenizer does not make use of special tokens. The two sequences have been concatenated.")
        return token_ids_0 + token_ids_1
870

871
872
873
874
875
876
877
878
    def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
        """ Converts a single index or a sequence of indices (integers) in a token "
            (resp.) a sequence of tokens (str/unicode), using the vocabulary and added tokens.

            Args:
                skip_special_tokens: Don't decode special tokens (self.all_special_tokens). Default: False
        """
        if isinstance(ids, int):
879
880
881
882
            if ids in self.added_tokens_decoder:
                return self.added_tokens_decoder[ids]
            else:
                return self._convert_id_to_token(ids)
883
884
        tokens = []
        for index in ids:
thomwolf's avatar
thomwolf committed
885
            if skip_special_tokens and index in self.all_special_ids:
886
887
888
889
890
891
892
893
                continue
            if index in self.added_tokens_decoder:
                tokens.append(self.added_tokens_decoder[index])
            else:
                tokens.append(self._convert_id_to_token(index))
        return tokens

    def _convert_id_to_token(self, index):
thomwolf's avatar
thomwolf committed
894
895
        raise NotImplementedError

896
897
898
899
    def convert_tokens_to_string(self, tokens):
        """ Converts a sequence of tokens (string) in a single string.
            The most simple way to do it is ' '.join(self.convert_ids_to_tokens(token_ids))
            but we often want to remove sub-word tokenization artifacts at the same time.
900
        """
901
        return ' '.join(self.convert_ids_to_tokens(tokens))
902
903

    def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
LysandreJik's avatar
Doc  
LysandreJik committed
904
905
906
        """
        Converts a sequence of ids (integer) in a string, using the tokenizer and vocabulary
        with options to remove special tokens and clean up tokenization spaces.
907
        Similar to doing ``self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))``.
908
909
        """
        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
thomwolf's avatar
thomwolf committed
910
911
912
913
914
915
916
917
918
919
920
921
922

        # To avoid mixing byte-level and unicode for byte-level BPT
        # we need to build string separatly for added tokens and byte-level tokens
        # cf. https://github.com/huggingface/pytorch-transformers/issues/1133
        sub_texts = []
        current_sub_text = []
        for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                continue
            if token in self.added_tokens_encoder:
                if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(current_sub_text))
                    current_sub_text = []
923
                sub_texts.append(" " + token)
thomwolf's avatar
thomwolf committed
924
925
926
927
928
            else:
                current_sub_text.append(token)
        if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(current_sub_text))
        text = ''.join(sub_texts)
929

930
931
932
        if self._sep_token is not None and self._sep_token in text:
            text = text.replace(self._cls_token, self._sep_token)
            split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self._sep_token)))
933
934
935
936
937
938
939
940
941
942
943
            if clean_up_tokenization_spaces:
                clean_text = [self.clean_up_tokenization(text) for text in split_text]
                return clean_text
            else:
                return split_text
        else:
            if clean_up_tokenization_spaces:
                clean_text = self.clean_up_tokenization(text)
                return clean_text
            else:
                return text
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964

    @property
    def special_tokens_map(self):
        """ A dictionary mapping special token class attribute (cls_token, unk_token...) to their
            values ('<unk>', '<cls>'...)
        """
        set_attr = {}
        for attr in self.SPECIAL_TOKENS_ATTRIBUTES:
            attr_value = getattr(self, "_" + attr)
            if attr_value:
                set_attr[attr] = attr_value
        return set_attr

    @property
    def all_special_tokens(self):
        """ List all the special tokens ('<unk>', '<cls>'...) mapped to class attributes
            (cls_token, unk_token...).
        """
        all_toks = []
        set_attr = self.special_tokens_map
        for attr_value in set_attr.values():
epwalsh's avatar
epwalsh committed
965
            all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value])
966
967
968
969
970
971
972
973
974
        all_toks = list(set(all_toks))
        return all_toks

    @property
    def all_special_ids(self):
        """ List the vocabulary indices of the special tokens ('<unk>', '<cls>'...) mapped to
            class attributes (cls_token, unk_token...).
        """
        all_toks = self.all_special_tokens
975
        all_ids = list(self._convert_token_to_id(t) for t in all_toks)
976
977
        return all_ids

thomwolf's avatar
thomwolf committed
978
979
    @staticmethod
    def clean_up_tokenization(out_string):
980
981
        """ Clean up a list of simple English tokenization artifacts like spaces before punctuations and abreviated forms.
        """
thomwolf's avatar
thomwolf committed
982
983
984
985
        out_string = out_string.replace(' .', '.').replace(' ?', '?').replace(' !', '!').replace(' ,', ','
                        ).replace(" ' ", "'").replace(" n't", "n't").replace(" 'm", "'m").replace(" do not", " don't"
                        ).replace(" 's", "'s").replace(" 've", "'ve").replace(" 're", "'re")
        return out_string