"docs_zh-CN/vscode:/vscode.git/clone" did not exist on "111f33be5aab4cae34e4f5601a2069421dd2c8e9"
tokenization_openai.py 11.8 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
thomwolf's avatar
thomwolf committed
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for OpenAI GPT."""
Aymeric Augustin's avatar
Aymeric Augustin committed
16

thomwolf's avatar
thomwolf committed
17
18
19

import json
import logging
thomwolf's avatar
thomwolf committed
20
import os
thomwolf's avatar
thomwolf committed
21
import re
22
23
24
25
26
27
28
29
30
from typing import List, Optional, Union

from tokenizers import Tokenizer
from tokenizers.decoders import BPEDecoder
from tokenizers.implementations import BaseTokenizer
from tokenizers.models import BPE
from tokenizers.normalizers import BertNormalizer, Sequence, unicode_normalizer_from_str
from tokenizers.pre_tokenizers import BertPreTokenizer
from tokenizers.trainers import BpeTrainer
thomwolf's avatar
thomwolf committed
31

thomwolf's avatar
thomwolf committed
32
from .tokenization_bert import BasicTokenizer
33
from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
Aymeric Augustin's avatar
Aymeric Augustin committed
34

thomwolf's avatar
thomwolf committed
35
36
37

logger = logging.getLogger(__name__)

38
VOCAB_FILES_NAMES = {
39
40
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
thomwolf's avatar
thomwolf committed
41
}
42
43

PRETRAINED_VOCAB_FILES_MAP = {
44
45
    "vocab_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json"},
    "merges_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"},
thomwolf's avatar
thomwolf committed
46
}
47
48

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
49
    "openai-gpt": 512,
thomwolf's avatar
thomwolf committed
50
}
thomwolf's avatar
thomwolf committed
51

52

thomwolf's avatar
thomwolf committed
53
54
55
56
57
58
59
60
61
62
63
64
def get_pairs(word):
    """
    Return set of symbol pairs in a word.
    word is represented as tuple of symbols (symbols being variable-length strings)
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

65

thomwolf's avatar
thomwolf committed
66
67
68
69
70
def text_standardize(text):
    """
    fixes some issues the spacy tokenizer had on books corpus
    also does some whitespace standardization
    """
71
72
73
74
75
76
77
78
    text = text.replace("—", "-")
    text = text.replace("–", "-")
    text = text.replace("―", "-")
    text = text.replace("…", "...")
    text = text.replace("´", "'")
    text = re.sub(r"""(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""", r" \1 ", text)
    text = re.sub(r"\s*\n\s*", " \n ", text)
    text = re.sub(r"[^\S\n]+", " ", text)
thomwolf's avatar
thomwolf committed
79
80
    return text.strip()

81

82
class OpenAIGPTTokenizer(PreTrainedTokenizer):
thomwolf's avatar
thomwolf committed
83
    """
84
    BPE tokenizer. Peculiarities:
Lysandre Debut's avatar
Lysandre Debut committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

    - lower case all inputs
    - uses SpaCy tokenizer and ftfy for pre-BPE tokenization if they are installed, fallback to BERT's BasicTokenizer if not.

    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Users
    should refer to the superclass for more information regarding methods.

    Args:
        vocab_file (:obj:`str`):
            Path to the vocabulary file.
        merges_file (:obj:`str`):
            Path to the merges file.
        unk_token (:obj:`string`, `optional`, defaults to "<unk>"):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
thomwolf's avatar
thomwolf committed
100
    """
101

102
103
104
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
thomwolf's avatar
thomwolf committed
105

106
    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
Julien Chaumond's avatar
Julien Chaumond committed
107
        super().__init__(unk_token=unk_token, **kwargs)
108

109
110
111
112
113
114
        self.max_len_single_sentence = (
            self.max_len
        )  # no default special tokens - you can update this value if you add special tokens
        self.max_len_sentences_pair = (
            self.max_len
        )  # no default special tokens - you can update this value if you add special tokens
115

thomwolf's avatar
thomwolf committed
116
117
        try:
            import ftfy
118
            from spacy.lang.en import English
119

120
            _nlp = English()
121
            self.nlp = _nlp.Defaults.create_tokenizer(_nlp)
122
            self.fix_text = ftfy.fix_text
thomwolf's avatar
thomwolf committed
123
        except ImportError:
124
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.")
125
            self.nlp = BasicTokenizer(do_lower_case=True)
126
            self.fix_text = None
thomwolf's avatar
thomwolf committed
127

128
129
        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
130
131
132
        self.decoder = {v: k for k, v in self.encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[1:-1]
thomwolf's avatar
thomwolf committed
133
134
135
        merges = [tuple(merge.split()) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}
136

137
138
139
    @property
    def vocab_size(self):
        return len(self.encoder)
thomwolf's avatar
thomwolf committed
140

141
142
143
    def get_vocab(self):
        return dict(self.encoder, **self.added_tokens_encoder)

thomwolf's avatar
thomwolf committed
144
    def bpe(self, token):
145
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
thomwolf's avatar
thomwolf committed
146
147
148
149
150
        if token in self.cache:
            return self.cache[token]
        pairs = get_pairs(word)

        if not pairs:
151
            return token + "</w>"
thomwolf's avatar
thomwolf committed
152
153

        while True:
154
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
thomwolf's avatar
thomwolf committed
155
156
157
158
159
160
161
162
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
163
                except ValueError:
thomwolf's avatar
thomwolf committed
164
165
                    new_word.extend(word[i:])
                    break
166
167
168
                else:
                    new_word.extend(word[i:j])
                    i = j
thomwolf's avatar
thomwolf committed
169

170
171
                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
thomwolf's avatar
thomwolf committed
172
173
174
175
176
177
178
179
180
181
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
182
183
184
        word = " ".join(word)
        if word == "\n  </w>":
            word = "\n</w>"
thomwolf's avatar
thomwolf committed
185
186
187
        self.cache[token] = word
        return word

188
    def _tokenize(self, text):
thomwolf's avatar
thomwolf committed
189
        """ Tokenize a string. """
thomwolf's avatar
thomwolf committed
190
        split_tokens = []
191
192
193
194
        if self.fix_text is None:
            # Using BERT's BasicTokenizer
            text = self.nlp.tokenize(text)
            for token in text:
195
                split_tokens.extend([t for t in self.bpe(token).split(" ")])
196
197
198
199
        else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            text = self.nlp(text_standardize(self.fix_text(text)))
            for token in text:
200
                split_tokens.extend([t for t in self.bpe(token.text.lower()).split(" ")])
thomwolf's avatar
thomwolf committed
201
202
        return split_tokens

203
    def _convert_token_to_id(self, token):
Aymeric Augustin's avatar
Aymeric Augustin committed
204
        """ Converts a token (str) in an id using the vocab. """
205
        return self.encoder.get(token, self.encoder.get(self.unk_token))
thomwolf's avatar
thomwolf committed
206

207
208
209
    def _convert_id_to_token(self, index):
        """Converts an id in a token (BPE) using the vocab."""
        return self.decoder.get(index, self.unk_token)
210

211
212
    def convert_tokens_to_string(self, tokens):
        """ Converts a sequence of tokens (string) in a single string. """
213
        out_string = "".join(tokens).replace("</w>", " ").strip()
thomwolf's avatar
thomwolf committed
214
        return out_string
215

216
    def save_vocabulary(self, save_directory):
Lysandre Debut's avatar
Lysandre Debut committed
217
218
219
220
221
222
223
224
225
226
        """
        Save the vocabulary and special tokens file to a directory.

        Args:
            save_directory (:obj:`str`):
                The directory in which to save the vocabulary.

        Returns:
            :obj:`Tuple(str)`: Paths to the files saved.
        """
227
228
        if not os.path.isdir(save_directory):
            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
229
            return
230
231
        vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_file"])
        merge_file = os.path.join(save_directory, VOCAB_FILES_NAMES["merges_file"])
thomwolf's avatar
thomwolf committed
232

233
        with open(vocab_file, "w", encoding="utf-8") as f:
thomwolf's avatar
thomwolf committed
234
235
            f.write(json.dumps(self.encoder, ensure_ascii=False))

236
237
        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
238
            writer.write("#version: 0.2\n")
239
240
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
241
242
243
244
                    logger.warning(
                        "Saving vocabulary to {}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!".format(merge_file)
                    )
245
                    index = token_index
246
                writer.write(" ".join(bpe_tokens) + "\n")
247
                index += 1
thomwolf's avatar
thomwolf committed
248

249
        return vocab_file, merge_file
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339


class _OpenAIGPTCharBPETokenizer(BaseTokenizer):
    """
    OpenAI character-level BPE Tokenizer
    """

    def __init__(
        self,
        vocab_file: Optional[str] = None,
        merges_file: Optional[str] = None,
        unk_token: Optional[str] = "<unk>",
        suffix: Optional[str] = "</w>",
        dropout: Optional[float] = None,
        unicode_normalizer: Optional[str] = None,
    ):
        if vocab_file is not None and merges_file is not None:
            tokenizer = Tokenizer(
                BPE.from_files(
                    vocab_file, merges_file, dropout=dropout, unk_token=unk_token, end_of_word_suffix=suffix
                )
            )
        else:
            tokenizer = Tokenizer(BPE.empty())

        # Check for Unicode normalization first (before everything else)
        normalizers = []

        if unicode_normalizer:
            normalizers += [unicode_normalizer_from_str(unicode_normalizer)]

        # OpenAI normalization is the same as Bert
        normalizers += [BertNormalizer()]

        # Create the normalizer structure
        if len(normalizers) > 0:
            if len(normalizers) > 1:
                tokenizer.normalizer = Sequence(normalizers)
            else:
                tokenizer.normalizer = normalizers[0]

        tokenizer.pre_tokenizer = BertPreTokenizer()
        tokenizer.decoder = BPEDecoder(suffix=suffix)

        parameters = {
            "model": "BPE",
            "unk_token": unk_token,
            "suffix": suffix,
            "dropout": dropout,
        }

        super().__init__(tokenizer, parameters)

    def train(
        self,
        files: Union[str, List[str]],
        vocab_size: int = 30000,
        min_frequency: int = 2,
        special_tokens: List[str] = ["<unk>"],
        limit_alphabet: int = 1000,
        initial_alphabet: List[str] = [],
        suffix: Optional[str] = "</w>",
        show_progress: bool = True,
    ):
        """ Train the model using the given files """

        trainer = BpeTrainer(
            vocab_size=vocab_size,
            min_frequency=min_frequency,
            special_tokens=special_tokens,
            limit_alphabet=limit_alphabet,
            initial_alphabet=initial_alphabet,
            end_of_word_suffix=suffix,
            show_progress=show_progress,
        )
        if isinstance(files, str):
            files = [files]
        self._tokenizer.train(trainer, files)


class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
        kwargs.setdefault("unk_token", unk_token)
        super().__init__(
            _OpenAIGPTCharBPETokenizer(vocab_file=vocab_file, merges_file=merges_file, unk_token=unk_token), **kwargs
        )