tokenization_openai.py 10.9 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
thomwolf's avatar
thomwolf committed
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for OpenAI GPT."""
Aymeric Augustin's avatar
Aymeric Augustin committed
16

thomwolf's avatar
thomwolf committed
17
18
19

import json
import logging
thomwolf's avatar
thomwolf committed
20
import os
thomwolf's avatar
thomwolf committed
21
import re
22
23
24
25
26
27
28
29
30
from typing import List, Optional, Union

from tokenizers import Tokenizer
from tokenizers.decoders import BPEDecoder
from tokenizers.implementations import BaseTokenizer
from tokenizers.models import BPE
from tokenizers.normalizers import BertNormalizer, Sequence, unicode_normalizer_from_str
from tokenizers.pre_tokenizers import BertPreTokenizer
from tokenizers.trainers import BpeTrainer
thomwolf's avatar
thomwolf committed
31

thomwolf's avatar
thomwolf committed
32
from .tokenization_bert import BasicTokenizer
33
from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
Aymeric Augustin's avatar
Aymeric Augustin committed
34

thomwolf's avatar
thomwolf committed
35
36
37

logger = logging.getLogger(__name__)

38
VOCAB_FILES_NAMES = {
39
40
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
thomwolf's avatar
thomwolf committed
41
}
42
43

PRETRAINED_VOCAB_FILES_MAP = {
44
45
    "vocab_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json"},
    "merges_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"},
thomwolf's avatar
thomwolf committed
46
}
47
48

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
49
    "openai-gpt": 512,
thomwolf's avatar
thomwolf committed
50
}
thomwolf's avatar
thomwolf committed
51

52

thomwolf's avatar
thomwolf committed
53
54
55
56
57
58
59
60
61
62
63
64
def get_pairs(word):
    """
    Return set of symbol pairs in a word.
    word is represented as tuple of symbols (symbols being variable-length strings)
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

65

thomwolf's avatar
thomwolf committed
66
67
68
69
70
def text_standardize(text):
    """
    fixes some issues the spacy tokenizer had on books corpus
    also does some whitespace standardization
    """
71
72
73
74
75
76
77
78
    text = text.replace("—", "-")
    text = text.replace("–", "-")
    text = text.replace("―", "-")
    text = text.replace("…", "...")
    text = text.replace("´", "'")
    text = re.sub(r"""(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""", r" \1 ", text)
    text = re.sub(r"\s*\n\s*", " \n ", text)
    text = re.sub(r"[^\S\n]+", " ", text)
thomwolf's avatar
thomwolf committed
79
80
    return text.strip()

81

82
class OpenAIGPTTokenizer(PreTrainedTokenizer):
thomwolf's avatar
thomwolf committed
83
    """
84
85
    BPE tokenizer. Peculiarities:
        - lower case all inputs
86
        - uses SpaCy tokenizer and ftfy for pre-BPE tokenization if they are installed, fallback to BERT's BasicTokenizer if not.
thomwolf's avatar
thomwolf committed
87
    """
88

89
90
91
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
thomwolf's avatar
thomwolf committed
92

93
    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
Julien Chaumond's avatar
Julien Chaumond committed
94
        super().__init__(unk_token=unk_token, **kwargs)
95

96
97
98
99
100
101
        self.max_len_single_sentence = (
            self.max_len
        )  # no default special tokens - you can update this value if you add special tokens
        self.max_len_sentences_pair = (
            self.max_len
        )  # no default special tokens - you can update this value if you add special tokens
102

thomwolf's avatar
thomwolf committed
103
104
        try:
            import ftfy
105
            from spacy.lang.en import English
106

107
            _nlp = English()
108
            self.nlp = _nlp.Defaults.create_tokenizer(_nlp)
109
            self.fix_text = ftfy.fix_text
thomwolf's avatar
thomwolf committed
110
        except ImportError:
111
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.")
112
            self.nlp = BasicTokenizer(do_lower_case=True)
113
            self.fix_text = None
thomwolf's avatar
thomwolf committed
114

115
116
        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
117
118
119
        self.decoder = {v: k for k, v in self.encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[1:-1]
thomwolf's avatar
thomwolf committed
120
121
122
        merges = [tuple(merge.split()) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}
123

124
125
126
    @property
    def vocab_size(self):
        return len(self.encoder)
thomwolf's avatar
thomwolf committed
127
128

    def bpe(self, token):
129
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
thomwolf's avatar
thomwolf committed
130
131
132
133
134
        if token in self.cache:
            return self.cache[token]
        pairs = get_pairs(word)

        if not pairs:
135
            return token + "</w>"
thomwolf's avatar
thomwolf committed
136
137

        while True:
138
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
thomwolf's avatar
thomwolf committed
139
140
141
142
143
144
145
146
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
147
                except ValueError:
thomwolf's avatar
thomwolf committed
148
149
                    new_word.extend(word[i:])
                    break
150
151
152
                else:
                    new_word.extend(word[i:j])
                    i = j
thomwolf's avatar
thomwolf committed
153

154
155
                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
thomwolf's avatar
thomwolf committed
156
157
158
159
160
161
162
163
164
165
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
166
167
168
        word = " ".join(word)
        if word == "\n  </w>":
            word = "\n</w>"
thomwolf's avatar
thomwolf committed
169
170
171
        self.cache[token] = word
        return word

172
    def _tokenize(self, text):
thomwolf's avatar
thomwolf committed
173
        """ Tokenize a string. """
thomwolf's avatar
thomwolf committed
174
        split_tokens = []
175
176
177
178
        if self.fix_text is None:
            # Using BERT's BasicTokenizer
            text = self.nlp.tokenize(text)
            for token in text:
179
                split_tokens.extend([t for t in self.bpe(token).split(" ")])
180
181
182
183
        else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            text = self.nlp(text_standardize(self.fix_text(text)))
            for token in text:
184
                split_tokens.extend([t for t in self.bpe(token.text.lower()).split(" ")])
thomwolf's avatar
thomwolf committed
185
186
        return split_tokens

187
    def _convert_token_to_id(self, token):
Aymeric Augustin's avatar
Aymeric Augustin committed
188
        """ Converts a token (str) in an id using the vocab. """
189
        return self.encoder.get(token, self.encoder.get(self.unk_token))
thomwolf's avatar
thomwolf committed
190

191
192
193
    def _convert_id_to_token(self, index):
        """Converts an id in a token (BPE) using the vocab."""
        return self.decoder.get(index, self.unk_token)
194

195
196
    def convert_tokens_to_string(self, tokens):
        """ Converts a sequence of tokens (string) in a single string. """
197
        out_string = "".join(tokens).replace("</w>", " ").strip()
thomwolf's avatar
thomwolf committed
198
        return out_string
199

200
    def save_vocabulary(self, save_directory):
201
        """Save the tokenizer vocabulary and merge files to a directory."""
202
203
        if not os.path.isdir(save_directory):
            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
204
            return
205
206
        vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_file"])
        merge_file = os.path.join(save_directory, VOCAB_FILES_NAMES["merges_file"])
thomwolf's avatar
thomwolf committed
207

208
        with open(vocab_file, "w", encoding="utf-8") as f:
thomwolf's avatar
thomwolf committed
209
210
            f.write(json.dumps(self.encoder, ensure_ascii=False))

211
212
        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
213
            writer.write("#version: 0.2\n")
214
215
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
216
217
218
219
                    logger.warning(
                        "Saving vocabulary to {}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!".format(merge_file)
                    )
220
                    index = token_index
221
                writer.write(" ".join(bpe_tokens) + "\n")
222
                index += 1
thomwolf's avatar
thomwolf committed
223

224
        return vocab_file, merge_file
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314


class _OpenAIGPTCharBPETokenizer(BaseTokenizer):
    """
    OpenAI character-level BPE Tokenizer
    """

    def __init__(
        self,
        vocab_file: Optional[str] = None,
        merges_file: Optional[str] = None,
        unk_token: Optional[str] = "<unk>",
        suffix: Optional[str] = "</w>",
        dropout: Optional[float] = None,
        unicode_normalizer: Optional[str] = None,
    ):
        if vocab_file is not None and merges_file is not None:
            tokenizer = Tokenizer(
                BPE.from_files(
                    vocab_file, merges_file, dropout=dropout, unk_token=unk_token, end_of_word_suffix=suffix
                )
            )
        else:
            tokenizer = Tokenizer(BPE.empty())

        # Check for Unicode normalization first (before everything else)
        normalizers = []

        if unicode_normalizer:
            normalizers += [unicode_normalizer_from_str(unicode_normalizer)]

        # OpenAI normalization is the same as Bert
        normalizers += [BertNormalizer()]

        # Create the normalizer structure
        if len(normalizers) > 0:
            if len(normalizers) > 1:
                tokenizer.normalizer = Sequence(normalizers)
            else:
                tokenizer.normalizer = normalizers[0]

        tokenizer.pre_tokenizer = BertPreTokenizer()
        tokenizer.decoder = BPEDecoder(suffix=suffix)

        parameters = {
            "model": "BPE",
            "unk_token": unk_token,
            "suffix": suffix,
            "dropout": dropout,
        }

        super().__init__(tokenizer, parameters)

    def train(
        self,
        files: Union[str, List[str]],
        vocab_size: int = 30000,
        min_frequency: int = 2,
        special_tokens: List[str] = ["<unk>"],
        limit_alphabet: int = 1000,
        initial_alphabet: List[str] = [],
        suffix: Optional[str] = "</w>",
        show_progress: bool = True,
    ):
        """ Train the model using the given files """

        trainer = BpeTrainer(
            vocab_size=vocab_size,
            min_frequency=min_frequency,
            special_tokens=special_tokens,
            limit_alphabet=limit_alphabet,
            initial_alphabet=initial_alphabet,
            end_of_word_suffix=suffix,
            show_progress=show_progress,
        )
        if isinstance(files, str):
            files = [files]
        self._tokenizer.train(trainer, files)


class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
        kwargs.setdefault("unk_token", unk_token)
        super().__init__(
            _OpenAIGPTCharBPETokenizer(vocab_file=vocab_file, merges_file=merges_file, unk_token=unk_token), **kwargs
        )