modeling_utils.py 69.3 KB
Newer Older
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team.
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch BERT model."""


import logging
import os

import torch
from torch import nn
24
25
from torch.nn import CrossEntropyLoss
from torch.nn import functional as F
26

27
from .configuration_utils import PretrainedConfig
28
from .file_utils import (
Aymeric Augustin's avatar
Aymeric Augustin committed
29
    DUMMY_INPUTS,
30
31
32
33
34
35
36
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    WEIGHTS_NAME,
    cached_path,
    hf_bucket_url,
    is_remote_url,
)
37

Aymeric Augustin's avatar
Aymeric Augustin committed
38

39
40
logger = logging.getLogger(__name__)

thomwolf's avatar
thomwolf committed
41
42
43
44
45
46
47
try:
    from torch.nn import Identity
except ImportError:
    # Older PyTorch compatibility
    class Identity(nn.Module):
        r"""A placeholder identity operator that is argument-insensitive.
        """
48

thomwolf's avatar
thomwolf committed
49
50
51
52
53
54
        def __init__(self, *args, **kwargs):
            super(Identity, self).__init__()

        def forward(self, input):
            return input

55

56
class PreTrainedModel(nn.Module):
57
58
    r""" Base class for all models.

59
        :class:`~transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models
Julien Chaumond's avatar
Julien Chaumond committed
60
        as well as a few methods common to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads.
61
62

        Class attributes (overridden by derived classes):
63
            - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture.
64
65
66
            - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values.
            - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments:

67
68
                - ``model``: an instance of the relevant subclass of :class:`~transformers.PreTrainedModel`,
                - ``config``: an instance of the relevant subclass of :class:`~transformers.PretrainedConfig`,
69
70
71
                - ``path``: a path (string) to the TensorFlow checkpoint.

            - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model.
72
    """
73
    config_class = None
74
75
76
    pretrained_model_archive_map = {}
    base_model_prefix = ""

77
78
79
80
81
82
83
    @property
    def dummy_inputs(self):
        """ Dummy inputs to do a forward pass in the network.

        Returns:
            torch.Tensor with dummy inputs
        """
84
        return {"input_ids": torch.tensor(DUMMY_INPUTS)}
85

86
87
88
89
90
91
92
93
    def __init__(self, config, *inputs, **kwargs):
        super(PreTrainedModel, self).__init__()
        if not isinstance(config, PretrainedConfig):
            raise ValueError(
                "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. "
                "To create a model from a pretrained model use "
                "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format(
                    self.__class__.__name__, self.__class__.__name__
94
95
                )
            )
thomwolf's avatar
thomwolf committed
96
        # Save config in model
97
98
        self.config = config

99
100
101
    @property
    def base_model(self):
        return getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
102

thomwolf's avatar
thomwolf committed
103
104
    def get_input_embeddings(self):
        """ Get model's input embeddings
thomwolf's avatar
thomwolf committed
105
        """
106
        base_model = getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
107
108
109
110
        if base_model is not self:
            return base_model.get_input_embeddings()
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
111

thomwolf's avatar
thomwolf committed
112
113
114
115
116
117
118
119
    def set_input_embeddings(self, value):
        """ Set model's input embeddings
        """
        base_model = getattr(self, self.base_model_prefix, self)
        if base_model is not self:
            base_model.set_input_embeddings(value)
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
120

thomwolf's avatar
thomwolf committed
121
122
123
124
    def get_output_embeddings(self):
        """ Get model's output embeddings
            Return None if the model doesn't have output embeddings
        """
125
        return None  # Overwrite for models with output embeddings
thomwolf's avatar
thomwolf committed
126

127
128
129
    def tie_weights(self):
        """ Make sure we are sharing the input and output embeddings.
            Export to TorchScript can't handle parameter sharing so we are cloning them instead.
thomwolf's avatar
thomwolf committed
130
        """
thomwolf's avatar
thomwolf committed
131
132
133
        output_embeddings = self.get_output_embeddings()
        if output_embeddings is not None:
            self._tie_or_clone_weights(output_embeddings, self.get_input_embeddings())
thomwolf's avatar
thomwolf committed
134

135
    def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
thomwolf's avatar
thomwolf committed
136
137
138
        """ Tie or clone module weights depending of weither we are using TorchScript or not
        """
        if self.config.torchscript:
139
            output_embeddings.weight = nn.Parameter(input_embeddings.weight.clone())
thomwolf's avatar
thomwolf committed
140
        else:
141
            output_embeddings.weight = input_embeddings.weight
thomwolf's avatar
thomwolf committed
142

143
        if hasattr(output_embeddings, "bias") and output_embeddings.bias is not None:
144
145
146
            output_embeddings.bias.data = torch.nn.functional.pad(
                output_embeddings.bias.data,
                (0, output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0]),
147
148
                "constant",
                0,
149
            )
150
        if hasattr(output_embeddings, "out_features") and hasattr(input_embeddings, "num_embeddings"):
151
            output_embeddings.out_features = input_embeddings.num_embeddings
152

thomwolf's avatar
thomwolf committed
153
154
    def resize_token_embeddings(self, new_num_tokens=None):
        """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size.
155
        Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
thomwolf's avatar
thomwolf committed
156

157
158
159
        Arguments:

            new_num_tokens: (`optional`) int:
160
                New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end.
161
                If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model.
thomwolf's avatar
thomwolf committed
162

thomwolf's avatar
thomwolf committed
163
        Return: ``torch.nn.Embeddings``
164
            Pointer to the input tokens Embeddings Module of the model
thomwolf's avatar
thomwolf committed
165
166
        """
        base_model = getattr(self, self.base_model_prefix, self)  # get the base model if needed
thomwolf's avatar
thomwolf committed
167
168
169
        model_embeds = base_model._resize_token_embeddings(new_num_tokens)
        if new_num_tokens is None:
            return model_embeds
thomwolf's avatar
thomwolf committed
170
171
172
173
174
175

        # Update base model and current model config
        self.config.vocab_size = new_num_tokens
        base_model.vocab_size = new_num_tokens

        # Tie weights again if needed
176
        self.tie_weights()
thomwolf's avatar
thomwolf committed
177

thomwolf's avatar
thomwolf committed
178
179
        return model_embeds

180
    def _resize_token_embeddings(self, new_num_tokens):
thomwolf's avatar
thomwolf committed
181
182
183
184
        old_embeddings = self.get_input_embeddings()
        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens)
        self.set_input_embeddings(new_embeddings)
        return self.get_input_embeddings()
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219

    def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None):
        """ Build a resized Embedding Module from a provided token Embedding Module.
            Increasing the size will add newly initialized vectors at the end
            Reducing the size will remove vectors from the end

        Args:
            new_num_tokens: (`optional`) int
                New number of tokens in the embedding matrix.
                Increasing the size will add newly initialized vectors at the end
                Reducing the size will remove vectors from the end
                If not provided or None: return the provided token Embedding Module.
        Return: ``torch.nn.Embeddings``
            Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None
        """
        if new_num_tokens is None:
            return old_embeddings

        old_num_tokens, old_embedding_dim = old_embeddings.weight.size()
        if old_num_tokens == new_num_tokens:
            return old_embeddings

        # Build new embeddings
        new_embeddings = nn.Embedding(new_num_tokens, old_embedding_dim)
        new_embeddings.to(old_embeddings.weight.device)

        # initialize all new embeddings (in particular added tokens)
        self._init_weights(new_embeddings)

        # Copy word embeddings from the previous weights
        num_tokens_to_copy = min(old_num_tokens, new_num_tokens)
        new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :]

        return new_embeddings

220
221
222
223
224
225
226
227
228
    def init_weights(self):
        """ Initialize and prunes weights if needed. """
        # Initialize weights
        self.apply(self._init_weights)

        # Prune heads if needed
        if self.config.pruned_heads:
            self.prune_heads(self.config.pruned_heads)

229
230
231
        # Tie weights if needed
        self.tie_weights()

thomwolf's avatar
thomwolf committed
232
233
    def prune_heads(self, heads_to_prune):
        """ Prunes heads of the base model.
234
235
236
237

            Arguments:

                heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`).
238
                E.g. {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2.
thomwolf's avatar
thomwolf committed
239
        """
240
        # save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads
241
        for layer, heads in heads_to_prune.items():
242
243
244
            union_heads = set(self.config.pruned_heads.get(layer, [])) | set(heads)
            self.config.pruned_heads[layer] = list(union_heads)  # Unfortunately we have to store it as list for JSON

245
        self.base_model._prune_heads(heads_to_prune)
thomwolf's avatar
thomwolf committed
246

247
    def save_pretrained(self, save_directory):
248
        """ Save a model and its configuration file to a directory, so that it
249
            can be re-loaded using the `:func:`~transformers.PreTrainedModel.from_pretrained`` class method.
250
        """
251
252
253
        assert os.path.isdir(
            save_directory
        ), "Saving path should be a directory where the model and configuration can be saved"
254

Julien Chaumond's avatar
Julien Chaumond committed
255
        # Only save the model itself if we are using distributed training
256
        model_to_save = self.module if hasattr(self, "module") else self
257

thomwolf's avatar
thomwolf committed
258
259
260
        # Save configuration file
        model_to_save.config.save_pretrained(save_directory)

261
262
263
        # If we save using the predefined names, we can load using `from_pretrained`
        output_model_file = os.path.join(save_directory, WEIGHTS_NAME)
        torch.save(model_to_save.state_dict(), output_model_file)
thomwolf's avatar
thomwolf committed
264
        logger.info("Model weights saved in {}".format(output_model_file))
265

266
    @classmethod
267
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
268
269
        r"""Instantiate a pretrained pytorch model from a pre-trained model configuration.

270
271
272
        The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated)
        To train the model, you should first set it back in training mode with ``model.train()``

273
274
275
276
277
        The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model.
        It is up to you to train those weights with a downstream fine-tuning task.

        The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded.

278
279
280
281
        Parameters:
            pretrained_model_name_or_path: either:

                - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
282
                - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``.
283
                - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
284
                - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
thomwolf's avatar
thomwolf committed
285
                - None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``)
286
287
288
289

            model_args: (`optional`) Sequence of positional arguments:
                All remaning positional arguments will be passed to the underlying model's ``__init__`` method

290
291
292
            config: (`optional`) one of:
                    - an instance of a class derived from :class:`~transformers.PretrainedConfig`, or
                    - a string valid as input to :func:`~transformers.PretrainedConfig.from_pretrained()`
293
294
295
                Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when:

                - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or
296
                - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory.
297
298
299
300
                - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory.

            state_dict: (`optional`) dict:
                an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file.
thomwolf's avatar
typos  
thomwolf committed
301
                This option can be used if you want to create a model from a pretrained configuration but load your own weights.
302
                In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option.
303
304

            cache_dir: (`optional`) string:
thomwolf's avatar
thomwolf committed
305
306
                Path to a directory in which a downloaded pre-trained model
                configuration should be cached if the standard cache should not be used.
307

308
309
310
            force_download: (`optional`) boolean, default False:
                Force to (re-)download the model weights and configuration files and override the cached versions if they exists.

311
312
313
            resume_download: (`optional`) boolean, default False:
                Do not delete incompletely recieved file. Attempt to resume the download if such a file exists.

314
315
316
317
            proxies: (`optional`) dict, default None:
                A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
                The proxies are used on each request.

318
            output_loading_info: (`optional`) boolean:
thomwolf's avatar
thomwolf committed
319
                Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
320
321
322
323
324

            kwargs: (`optional`) Remaining dictionary of keyword arguments:
                Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:

                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
325
                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
326
327

        Examples::
thomwolf's avatar
thomwolf committed
328

thomwolf's avatar
thomwolf committed
329
330
331
332
333
334
335
            model = BertModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
            model = BertModel.from_pretrained('./test/saved_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
            model = BertModel.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
            assert model.config.output_attention == True
            # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json')
            model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config)
thomwolf's avatar
thomwolf committed
336

337
        """
338
339
340
341
342
343
344
345
        config = kwargs.pop("config", None)
        state_dict = kwargs.pop("state_dict", None)
        cache_dir = kwargs.pop("cache_dir", None)
        from_tf = kwargs.pop("from_tf", False)
        force_download = kwargs.pop("force_download", False)
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        output_loading_info = kwargs.pop("output_loading_info", False)
thomwolf's avatar
thomwolf committed
346

347
348
349
        # Load config if we don't provide a configuration
        if not isinstance(config, PretrainedConfig):
            config_path = config if config is not None else pretrained_model_name_or_path
350
            config, model_kwargs = cls.config_class.from_pretrained(
351
352
353
354
                config_path,
                *model_args,
                cache_dir=cache_dir,
                return_unused_kwargs=True,
355
                force_download=force_download,
356
                resume_download=resume_download,
357
                proxies=proxies,
358
                **kwargs
359
360
361
            )
        else:
            model_kwargs = kwargs
362

thomwolf's avatar
thomwolf committed
363
        # Load model
thomwolf's avatar
thomwolf committed
364
        if pretrained_model_name_or_path is not None:
365
            if pretrained_model_name_or_path in cls.pretrained_model_archive_map:
thomwolf's avatar
thomwolf committed
366
367
                archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path]
            elif os.path.isdir(pretrained_model_name_or_path):
thomwolf's avatar
thomwolf committed
368
369
                if from_tf and os.path.isfile(os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index")):
                    # Load from a TF 1.0 checkpoint
thomwolf's avatar
thomwolf committed
370
                    archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index")
thomwolf's avatar
thomwolf committed
371
372
373
374
375
                elif from_tf and os.path.isfile(os.path.join(pretrained_model_name_or_path, TF2_WEIGHTS_NAME)):
                    # Load from a TF 2.0 checkpoint
                    archive_file = os.path.join(pretrained_model_name_or_path, TF2_WEIGHTS_NAME)
                elif os.path.isfile(os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME)):
                    # Load from a PyTorch checkpoint
thomwolf's avatar
thomwolf committed
376
                    archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME)
thomwolf's avatar
thomwolf committed
377
                else:
378
379
380
381
382
                    raise EnvironmentError(
                        "Error no file named {} found in directory {} or `from_tf` set to False".format(
                            [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"], pretrained_model_name_or_path
                        )
                    )
383
            elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
384
                archive_file = pretrained_model_name_or_path
385
            elif os.path.isfile(pretrained_model_name_or_path + ".index"):
386
387
388
389
390
                assert (
                    from_tf
                ), "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format(
                    pretrained_model_name_or_path + ".index"
                )
391
                archive_file = pretrained_model_name_or_path + ".index"
392
            else:
393
                archive_file = hf_bucket_url(pretrained_model_name_or_path, postfix=WEIGHTS_NAME)
Julien Chaumond's avatar
Julien Chaumond committed
394
                if from_tf:
395
396
397
                    raise EnvironmentError(
                        "Loading a PyTorch model from a TF checkpoint is not supported when using a model identifier name."
                    )
398

thomwolf's avatar
thomwolf committed
399
400
            # redirect to the cache, if necessary
            try:
401
402
403
404
405
406
407
                resolved_archive_file = cached_path(
                    archive_file,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                )
thomwolf's avatar
thomwolf committed
408
            except EnvironmentError:
thomwolf's avatar
thomwolf committed
409
                if pretrained_model_name_or_path in cls.pretrained_model_archive_map:
410
                    msg = "Couldn't reach server at '{}' to download pretrained weights.".format(archive_file)
thomwolf's avatar
thomwolf committed
411
                else:
412
413
414
                    msg = (
                        "Model name '{}' was not found in model name list ({}). "
                        "We assumed '{}' was a path or url to model weight files named one of {} but "
thomwolf's avatar
thomwolf committed
415
                        "couldn't find any such file at this path or url.".format(
thomwolf's avatar
thomwolf committed
416
                            pretrained_model_name_or_path,
417
                            ", ".join(cls.pretrained_model_archive_map.keys()),
thomwolf's avatar
thomwolf committed
418
                            archive_file,
419
420
421
                            [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME],
                        )
                    )
thomwolf's avatar
thomwolf committed
422
423
                raise EnvironmentError(msg)

thomwolf's avatar
thomwolf committed
424
425
            if resolved_archive_file == archive_file:
                logger.info("loading weights file {}".format(archive_file))
426
            else:
427
                logger.info("loading weights file {} from cache at {}".format(archive_file, resolved_archive_file))
428
        else:
thomwolf's avatar
thomwolf committed
429
            resolved_archive_file = None
430
431

        # Instantiate model.
432
        model = cls(config, *model_args, **model_kwargs)
thomwolf's avatar
thomwolf committed
433

434
        if state_dict is None and not from_tf:
435
            try:
436
                state_dict = torch.load(resolved_archive_file, map_location="cpu")
437
            except Exception:
438
439
440
441
                raise OSError(
                    "Unable to load weights from pytorch checkpoint file. "
                    "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. "
                )
442

443
444
445
        missing_keys = []
        unexpected_keys = []
        error_msgs = []
446
447

        if from_tf:
448
            if resolved_archive_file.endswith(".index"):
449
450
451
452
453
                # Load from a TensorFlow 1.X checkpoint - provided by original authors
                model = cls.load_tf_weights(model, config, resolved_archive_file[:-6])  # Remove the '.index'
            else:
                # Load from our TensorFlow 2.0 checkpoints
                try:
454
                    from transformers import load_tf2_checkpoint_in_pytorch_model
455

456
                    model = load_tf2_checkpoint_in_pytorch_model(model, resolved_archive_file, allow_missing_keys=True)
457
                except ImportError:
458
459
460
461
                    logger.error(
                        "Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see "
                        "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions."
                    )
462
                    raise
463
464
465
466
467
468
        else:
            # Convert old format to new format if needed from a PyTorch state_dict
            old_keys = []
            new_keys = []
            for key in state_dict.keys():
                new_key = None
469
470
471
472
                if "gamma" in key:
                    new_key = key.replace("gamma", "weight")
                if "beta" in key:
                    new_key = key.replace("beta", "bias")
473
474
475
476
477
478
479
                if new_key:
                    old_keys.append(key)
                    new_keys.append(new_key)
            for old_key, new_key in zip(old_keys, new_keys):
                state_dict[new_key] = state_dict.pop(old_key)

            # copy state_dict so _load_from_state_dict can modify it
480
            metadata = getattr(state_dict, "_metadata", None)
481
482
483
484
            state_dict = state_dict.copy()
            if metadata is not None:
                state_dict._metadata = metadata

485
486
            # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants
            # so we need to apply the function recursively.
487
            def load(module, prefix=""):
488
489
                local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
                module._load_from_state_dict(
490
491
                    state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs
                )
492
493
                for name, child in module._modules.items():
                    if child is not None:
494
                        load(child, prefix + name + ".")
495
496

            # Make sure we are able to load base models as well as derived models (with heads)
497
            start_prefix = ""
498
            model_to_load = model
499
500
501
502
503
504
505
            if not hasattr(model, cls.base_model_prefix) and any(
                s.startswith(cls.base_model_prefix) for s in state_dict.keys()
            ):
                start_prefix = cls.base_model_prefix + "."
            if hasattr(model, cls.base_model_prefix) and not any(
                s.startswith(cls.base_model_prefix) for s in state_dict.keys()
            ):
506
507
508
509
                model_to_load = getattr(model, cls.base_model_prefix)

            load(model_to_load, prefix=start_prefix)
            if len(missing_keys) > 0:
510
511
512
513
514
                logger.info(
                    "Weights of {} not initialized from pretrained model: {}".format(
                        model.__class__.__name__, missing_keys
                    )
                )
515
            if len(unexpected_keys) > 0:
516
517
518
519
520
                logger.info(
                    "Weights from pretrained model not used in {}: {}".format(
                        model.__class__.__name__, unexpected_keys
                    )
                )
521
            if len(error_msgs) > 0:
522
523
524
525
526
                raise RuntimeError(
                    "Error(s) in loading state_dict for {}:\n\t{}".format(
                        model.__class__.__name__, "\n\t".join(error_msgs)
                    )
                )
527

528
        model.tie_weights()  # make sure word embedding weights are still tied if needed
529

530
531
532
        # Set model in evaluation mode to desactivate DropOut modules by default
        model.eval()

thomwolf's avatar
thomwolf committed
533
534
535
536
        if output_loading_info:
            loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs}
            return model, loading_info

537
538
        return model

thomwolf's avatar
thomwolf committed
539
540
541
    def prepare_inputs_for_generation(self, input_ids, **kwargs):
        return {"input_ids": input_ids}

542
    def _do_output_past(self, outputs):
543
544
        # TODO: might be better to write a self.do_output_past method for each
        # individual class as is done for prepare_inputs_for_generation
545
546
        has_output_past = hasattr(self.config, 'output_past') and self.config.output_past
        has_multiple_outputs = len(outputs) > 1
547
        has_mem_len = hasattr(self.config, 'mem_len')
548
549

        if has_output_past and has_multiple_outputs and not has_mem_len:
550
551
552
553
            return True
        # TODO: Add cases for (xlnet, transfo_xl) using mem_len
        return False

thomwolf's avatar
thomwolf committed
554
    @torch.no_grad()
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
    def generate(
        self,
        input_ids=None,
        max_length=None,
        do_sample=None,
        num_beams=None,
        temperature=None,
        top_k=None,
        top_p=None,
        repetition_penalty=None,
        bos_token_id=None,
        pad_token_id=None,
        eos_token_ids=None,
        length_penalty=None,
        num_return_sequences=None,
    ):
thomwolf's avatar
thomwolf committed
571
572
573
574
        """ Sequence generator for models with a LM head.

        The method currently supports greedy or penalized greedy decoding, sampling with top-k or nucleus sampling
        and beam-search.
thomwolf's avatar
thomwolf committed
575

thomwolf's avatar
thomwolf committed
576
        Adapted in part from Facebook's XLM beam search code: https://github.com/facebookresearch/XLM
thomwolf's avatar
thomwolf committed
577
578
579
580
581

        Params:
            **input_ids**: (`optional`) `torch.LongTensor` of shape (1, sequence_length)
                The sequence used as a prompt for the generation. If `None` the method initializes
                it as an empty `torch.LongTensor` of shape (1,)
thomwolf's avatar
thomwolf committed
582
583
            **max_length**: (`optional`) int
                The max length of the sequence to be generated.  Between 1 and infinity. Default to 20.
thomwolf's avatar
thomwolf committed
584
            **do_sample**: (`optional`) bool
thomwolf's avatar
thomwolf committed
585
586
587
                If set to `False` we use greedy decoding; otherwise sampling. Default to greedy sampling.
            **num_beams**: (`optional`) int
                Number of beams for beam search. 1 means no beam serach. Default to 1.
thomwolf's avatar
thomwolf committed
588
589
            **temperature**: (`optional`) float
                The value used to module the next token probabilities.
thomwolf's avatar
thomwolf committed
590
591
592
593
            **top_k**: (`optional`) int
                The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.
            **top_p**: (`optional`) float
                The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.
thomwolf's avatar
thomwolf committed
594
            **repetition_penalty**: (`optional`) float
thomwolf's avatar
thomwolf committed
595
                The parameter for repetition penalty. Between 1.0 and + infinity. 1.0 means no penalty. Default to 1.
thomwolf's avatar
thomwolf committed
596
597
598
599
600
601
602
603
604
605
            **bos_token_id**: (`optional`) int
                Beginning of sentence token if no prompt is provided. Default to 0.
            **eos_token_ids**: (`optional`) int or list of int
                End of sequence token or list of tokens to stop the generation. Default to 0.
            **length_penalty**: (`optional`) int
                Exponential penalty to the length. Default to 0.
            **length_penalty**: (`optional`) float
                Exponential penalty to the length. Default to 1.
            **num_return_sequences**: (`optional`) int
                The number of independantly computed returned sequences for each element in the batch. Default to 1.
thomwolf's avatar
thomwolf committed
606
607
608
609
        """

        # We cannot generate if the model does not have a LM head
        if self.get_output_embeddings() is None:
610
611
612
613
            raise AttributeError(
                "You tried to generate sequences with a model that does not have a LM Head."
                "Please use another model class (e.g. `OpenAIGPTLMHeadModel`)"
            )
thomwolf's avatar
thomwolf committed
614

615
616
617
618
619
620
621
622
623
624
625
        max_length = max_length if max_length is not None else self.config.max_length
        do_sample = do_sample if do_sample is not None else self.config.do_sample
        num_beams = num_beams if num_beams is not None else self.config.num_beams
        temperature = temperature if temperature is not None else self.config.temperature
        top_k = top_k if top_k is not None else self.config.top_k
        top_p = top_p if top_p is not None else self.config.top_p
        repetition_penalty = repetition_penalty if repetition_penalty is not None else self.config.repetition_penalty
        bos_token_id = bos_token_id if bos_token_id is not None else self.config.bos_token_id
        pad_token_id = pad_token_id if pad_token_id is not None else self.config.pad_token_id
        eos_token_ids = eos_token_ids if eos_token_ids is not None else self.config.eos_token_ids
        length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty
626
627
628
        num_return_sequences = (
            num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
        )
thomwolf's avatar
thomwolf committed
629
630
631

        if input_ids is not None:
            batch_size = input_ids.shape[0]  # overriden by the input batch_size
thomwolf's avatar
thomwolf committed
632
633
        else:
            batch_size = 1
thomwolf's avatar
thomwolf committed
634
635
636
        if isinstance(eos_token_ids, int):
            eos_token_ids = [eos_token_ids]

thomwolf's avatar
thomwolf committed
637
        assert isinstance(max_length, int) and max_length > 0, "`max_length` should be a strictely positive integer."
thomwolf's avatar
thomwolf committed
638
        assert isinstance(do_sample, bool), "`do_sample` should be a boolean."
thomwolf's avatar
thomwolf committed
639
        assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictely positive integer."
640
        # assert temperature >= 0, "`temperature` should be positive."
641
        assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer."
thomwolf's avatar
thomwolf committed
642
        assert 0 <= top_p <= 1, "`top_p` should be between 0 and 1."
thomwolf's avatar
thomwolf committed
643
644
645
        assert repetition_penalty >= 1.0, "`repetition_penalty` should be >= 1."
        assert isinstance(bos_token_id, int) and bos_token_id >= 0, "`bos_token_id` should be a positive integer."
        assert isinstance(pad_token_id, int) and pad_token_id >= 0, "`pad_token_id` should be a positive integer."
646
647
648
        assert isinstance(eos_token_ids, (list, tuple)) and (
            e >= 0 for e in eos_token_ids
        ), "`eos_token_ids` should be a positive integer or a list/tuple of positive integers."
thomwolf's avatar
thomwolf committed
649
        assert length_penalty > 0, "`length_penalty` should be strictely positive."
650
651
652
        assert (
            isinstance(num_return_sequences, int) and num_return_sequences > 0
        ), "`num_return_sequences` should be a strictely positive integer."
thomwolf's avatar
thomwolf committed
653
654

        if input_ids is None:
655
656
657
            input_ids = torch.full(
                (batch_size, 1), bos_token_id, dtype=torch.long, device=next(self.parameters()).device
            )
thomwolf's avatar
thomwolf committed
658
        else:
659
            assert input_ids.dim() == 2, "Input prompt should be of shape (batch_size, sequence length)."
thomwolf's avatar
thomwolf committed
660
661

        # current position and vocab size
thomwolf's avatar
thomwolf committed
662
        cur_len = input_ids.shape[1]
thomwolf's avatar
thomwolf committed
663
664
        vocab_size = self.config.vocab_size

thomwolf's avatar
thomwolf committed
665
666
667
        if num_return_sequences != 1:
            # Expand input to num return sequences
            input_ids = input_ids.unsqueeze(1).expand(batch_size, num_return_sequences, cur_len)
668
669
670
            input_ids = input_ids.contiguous().view(
                batch_size * num_return_sequences, cur_len
            )  # (batch_size * num_return_sequences, cur_len)
thomwolf's avatar
thomwolf committed
671
672
673
674
            effective_batch_size = batch_size * num_return_sequences
        else:
            effective_batch_size = batch_size

thomwolf's avatar
thomwolf committed
675
        if num_beams > 1:
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
            output = self._generate_beam_search(
                input_ids,
                cur_len,
                max_length,
                do_sample,
                temperature,
                top_k,
                top_p,
                repetition_penalty,
                pad_token_id,
                eos_token_ids,
                effective_batch_size,
                length_penalty,
                num_beams,
                vocab_size,
            )
thomwolf's avatar
thomwolf committed
692
        else:
693
694
695
696
697
698
699
700
701
702
703
704
705
            output = self._generate_no_beam_search(
                input_ids,
                cur_len,
                max_length,
                do_sample,
                temperature,
                top_k,
                top_p,
                repetition_penalty,
                pad_token_id,
                eos_token_ids,
                effective_batch_size,
            )
thomwolf's avatar
thomwolf committed
706
707
708
709

        if num_return_sequences != 1:
            output = output.view(batch_size, num_return_sequences, -1)
        return output
thomwolf's avatar
thomwolf committed
710

711
712
713
714
715
716
717
718
719
720
721
722
723
724
    def _generate_no_beam_search(
        self,
        input_ids,
        cur_len,
        max_length,
        do_sample,
        temperature,
        top_k,
        top_p,
        repetition_penalty,
        pad_token_id,
        eos_token_ids,
        batch_size,
    ):
thomwolf's avatar
thomwolf committed
725
        """ Generate sequences for each example without beam search (num_beams == 1).
726
727
            All returned sequence are generated independantly.
        """
thomwolf's avatar
thomwolf committed
728
        # current position / max lengths / length of generated sentences / unfinished sentences
thomwolf's avatar
thomwolf committed
729
        unfinished_sents = input_ids.new(batch_size).fill_(1)
thomwolf's avatar
thomwolf committed
730

731
        past = None
thomwolf's avatar
thomwolf committed
732
733

        while cur_len < max_length:
734
            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
thomwolf's avatar
thomwolf committed
735
736
737
            outputs = self(**model_inputs)
            next_token_logits = outputs[0][:, -1, :]

patrickvonplaten's avatar
patrickvonplaten committed
738
            # if model has past, then set the past variable to speed up decoding
739
            if self._do_output_past(outputs):
740
741
                past = outputs[1]

thomwolf's avatar
thomwolf committed
742
743
            # repetition penalty from CTRL paper (https://arxiv.org/abs/1909.05858)
            if repetition_penalty != 1.0:
thomwolf's avatar
thomwolf committed
744
                for i in range(batch_size):
745
                    for previous_tokens in set(input_ids[i].tolist()):
746
747
748
749
750
                        # if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
                        if next_token_logits[i, previous_tokens] < 0:
                            next_token_logits[i, previous_tokens] *= repetition_penalty
                        else:
                            next_token_logits[i, previous_tokens] /= repetition_penalty
thomwolf's avatar
thomwolf committed
751
752
753

            if do_sample:
                # Temperature (higher temperature => more likely to sample low probability tokens)
754
                if temperature > 0 and temperature != 1.0:
thomwolf's avatar
thomwolf committed
755
756
757
758
                    next_token_logits = next_token_logits / temperature
                # Top-p/top-k filtering
                next_token_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)
                # Sample
759
                next_token = torch.multinomial(F.softmax(next_token_logits, dim=-1), num_samples=1).squeeze(1)
thomwolf's avatar
thomwolf committed
760
761
            else:
                # Greedy decoding
762
                next_token = torch.argmax(next_token_logits, dim=-1)
thomwolf's avatar
thomwolf committed
763
764
765

            # update generations and finished sentences
            tokens_to_add = next_token * unfinished_sents + pad_token_id * (1 - unfinished_sents)
766
            input_ids = torch.cat([input_ids, tokens_to_add.unsqueeze(-1)], dim=-1)
thomwolf's avatar
thomwolf committed
767
            for eos_token_id in eos_token_ids:
768
                unfinished_sents.mul_(tokens_to_add.ne(eos_token_id).long())
thomwolf's avatar
thomwolf committed
769
770
771
772
773
774
775
776
            cur_len = cur_len + 1

            # stop when there is a </s> in each sentence, or if we exceed the maximul length
            if unfinished_sents.max() == 0:
                break

        # add eos_token_ids to unfinished sentences
        if cur_len == max_length:
777
778
            input_ids[:, -1].masked_fill_(unfinished_sents.to(dtype=torch.bool), eos_token_ids[0])

thomwolf's avatar
thomwolf committed
779
780
        return input_ids

781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
    def _generate_beam_search(
        self,
        input_ids,
        cur_len,
        max_length,
        do_sample,
        temperature,
        top_k,
        top_p,
        repetition_penalty,
        pad_token_id,
        eos_token_ids,
        batch_size,
        length_penalty,
        num_beams,
        vocab_size,
    ):
thomwolf's avatar
thomwolf committed
798
        """ Generate sequences for each example with beam search.
799
        """
thomwolf's avatar
thomwolf committed
800
801
        # Expand input to num beams
        input_ids = input_ids.unsqueeze(1).expand(batch_size, num_beams, cur_len)
802
        input_ids = input_ids.contiguous().view(batch_size * num_beams, cur_len)  # (batch_size * num_beams, cur_len)
thomwolf's avatar
thomwolf committed
803
804

        # generated hypotheses
805
806
807
        generated_hyps = [
            BeamHypotheses(num_beams, max_length, length_penalty, early_stopping=False) for _ in range(batch_size)
        ]
thomwolf's avatar
thomwolf committed
808
809
810
811

        # scores for each sentence in the beam
        beam_scores = torch.zeros((batch_size, num_beams), dtype=torch.float, device=input_ids.device)
        beam_scores[:, 1:] = -1e9
812
        beam_scores = beam_scores.view(-1)  # shape (batch_size * num_beams,)
thomwolf's avatar
thomwolf committed
813
814

        # cache compute states
815
        past = None
thomwolf's avatar
thomwolf committed
816
817
818
819
820

        # done sentences
        done = [False for _ in range(batch_size)]

        while cur_len < max_length:
821
822
823
824
            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
            outputs = self(**model_inputs)  # (batch_size * num_beams, cur_len, vocab_size)
            scores = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)

patrickvonplaten's avatar
patrickvonplaten committed
825
            # if model has past, then set the past variable to speed up decoding
826
            if self._do_output_past(outputs):
827
                past = outputs[1]
thomwolf's avatar
thomwolf committed
828

829
830
831
832
            # repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858)
            if repetition_penalty != 1.0:
                for i in range(batch_size * num_beams):
                    for previous_tokens in set(input_ids[i].tolist()):
833
834
835
836
837
                        # if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
                        if scores[i, previous_tokens] < 0:
                            scores[i, previous_tokens] *= repetition_penalty
                        else:
                            scores[i, previous_tokens] /= repetition_penalty
thomwolf's avatar
thomwolf committed
838

839
840
            if do_sample:
                # Temperature (higher temperature => more likely to sample low probability tokens)
841
                if temperature > 0 and temperature != 1.0:
842
843
                    scores = scores / temperature
                # Top-p/top-k filtering
844
845
846
                scores = top_k_top_p_filtering(
                    scores, top_k=top_k, top_p=top_p, min_tokens_to_keep=2
                )  # (batch_size * num_beams, vocab_size)
847
                # Sample 2 next words for each beam (so we have some spare tokens and match output of greedy beam search)
848
                next_words = torch.multinomial(F.softmax(scores, dim=-1), num_samples=2)  # (batch_size * num_beams, 2)
849
                # Compute next scores
850
851
852
                _scores = F.log_softmax(scores, dim=-1)  # (batch_size * num_beams, vocab_size)
                _scores = torch.gather(_scores, -1, next_words)  # (batch_size * num_beams, 2)
                next_scores = _scores + beam_scores[:, None].expand_as(_scores)  # (batch_size * num_beams, 2)
853
                # Match shape of greedy beam search
854
855
                next_words = next_words.view(batch_size, 2 * num_beams)  # (batch_size, 2 * num_beams)
                next_scores = next_scores.view(batch_size, 2 * num_beams)  # (batch_size, 2 * num_beams)
856
857
            else:
                # do greedy beam search
858
                scores = F.log_softmax(scores, dim=-1)  # (batch_size * num_beams, vocab_size)
859
860
                assert scores.size() == (batch_size * num_beams, vocab_size)
                # Add the log prob of the new beams to the log prob of the beginning of the sequence (sum of logs == log of the product)
861
                _scores = scores + beam_scores[:, None].expand_as(scores)  # (batch_size * num_beams, vocab_size)
862
                # re-organize to group the beam together (we are keeping top hypothesis accross beams)
863
864
                _scores = _scores.view(batch_size, num_beams * vocab_size)  # (batch_size, num_beams * vocab_size)
                next_scores, next_words = torch.topk(_scores, 2 * num_beams, dim=1, largest=True, sorted=True)
thomwolf's avatar
thomwolf committed
865
866
867
868
869
870
871
872

            assert next_scores.size() == next_words.size() == (batch_size, 2 * num_beams)

            # next batch beam content
            # list of (batch_size * num_beams) tuple(next hypothesis score, next word, current position in the batch)
            next_batch_beam = []

            # for each sentence
thomwolf's avatar
thomwolf committed
873
            for batch_ex in range(batch_size):
thomwolf's avatar
thomwolf committed
874
875

                # if we are done with this sentence
thomwolf's avatar
thomwolf committed
876
877
                done[batch_ex] = done[batch_ex] or generated_hyps[batch_ex].is_done(next_scores[batch_ex].max().item())
                if done[batch_ex]:
thomwolf's avatar
thomwolf committed
878
879
880
881
882
883
884
                    next_batch_beam.extend([(0, pad_token_id, 0)] * num_beams)  # pad the batch
                    continue

                # next sentence beam content
                next_sent_beam = []

                # next words for this sentence
thomwolf's avatar
thomwolf committed
885
                for idx, score in zip(next_words[batch_ex], next_scores[batch_ex]):
thomwolf's avatar
thomwolf committed
886
887
888
889
890
891
892

                    # get beam and word IDs
                    beam_id = idx // vocab_size
                    word_id = idx % vocab_size

                    # end of sentence, or next word
                    if word_id.item() in eos_token_ids or cur_len + 1 == max_length:
893
894
895
                        generated_hyps[batch_ex].add(
                            input_ids[batch_ex * num_beams + beam_id, :cur_len].clone(), score.item()
                        )
thomwolf's avatar
thomwolf committed
896
                    else:
thomwolf's avatar
thomwolf committed
897
                        next_sent_beam.append((score, word_id, batch_ex * num_beams + beam_id))
thomwolf's avatar
thomwolf committed
898
899
900
901
902
903
904
905
906
907

                    # the beam for next step is full
                    if len(next_sent_beam) == num_beams:
                        break

                # update next beam content
                assert len(next_sent_beam) == 0 if cur_len + 1 == max_length else num_beams
                if len(next_sent_beam) == 0:
                    next_sent_beam = [(0, pad_token_id, 0)] * num_beams  # pad the batch
                next_batch_beam.extend(next_sent_beam)
thomwolf's avatar
thomwolf committed
908
                assert len(next_batch_beam) == num_beams * (batch_ex + 1)
thomwolf's avatar
thomwolf committed
909
910
911
912
913
914
915

            # sanity check / prepare next batch
            assert len(next_batch_beam) == batch_size * num_beams
            beam_scores = beam_scores.new([x[0] for x in next_batch_beam])
            beam_words = input_ids.new([x[1] for x in next_batch_beam])
            beam_idx = input_ids.new([x[2] for x in next_batch_beam])

916
            # re-order batch
thomwolf's avatar
thomwolf committed
917
918
            input_ids = input_ids[beam_idx, :]
            input_ids = torch.cat([input_ids, beam_words.unsqueeze(1)], dim=-1)
919
920
921
922
923
924
925

            # re-order internal states
            if past:
                reordered_past = []
                for layer_past in past:
                    # copy the relevant beam idx past to past
                    reordered_layer_past = [layer_past[:, i].unsqueeze(1).clone().detach() for i in beam_idx]
926
927
928
929
                    reordered_layer_past = torch.cat(reordered_layer_past, dim=1)
                    # check that shape matches
                    assert reordered_layer_past.shape == layer_past.shape
                    reordered_past.append(reordered_layer_past)
930
                past = tuple(reordered_past)
thomwolf's avatar
thomwolf committed
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948

            # update current length
            cur_len = cur_len + 1

            # stop when we are done with each sentence
            if all(done):
                break

        # visualize hypotheses
        # print([len(x) for x in generated_hyps], cur_len)
        # globals().update( locals() );
        # !import code; code.interact(local=vars())
        # for ii in range(batch_size):
        #     for ss, ww in sorted(generated_hyps[ii].hyp, key=lambda x: x[0], reverse=True):
        #         print("%.3f " % ss + " ".join(self.dico[x] for x in ww.tolist()))
        #     print("")

        # select the best hypotheses
thomwolf's avatar
thomwolf committed
949
950
        tgt_len = input_ids.new(batch_size)
        best = []
thomwolf's avatar
thomwolf committed
951
952

        for i, hypotheses in enumerate(generated_hyps):
thomwolf's avatar
thomwolf committed
953
954
955
            best_hyp = max(hypotheses.hyp, key=lambda x: x[0])[1]
            tgt_len[i] = len(best_hyp) + 1  # +1 for the <EOS> symbol
            best.append(best_hyp)
thomwolf's avatar
thomwolf committed
956
957

        # generate target batch
thomwolf's avatar
thomwolf committed
958
959
        decoded = input_ids.new(batch_size, tgt_len.max().item()).fill_(pad_token_id)
        for i, hypo in enumerate(best):
960
            decoded[i, : tgt_len[i] - 1] = hypo
thomwolf's avatar
thomwolf committed
961
            decoded[i, tgt_len[i] - 1] = eos_token_ids[0]
thomwolf's avatar
thomwolf committed
962

thomwolf's avatar
thomwolf committed
963
964
965
        return decoded


966
def top_k_top_p_filtering(logits, top_k=0, top_p=1.0, filter_value=-float("Inf"), min_tokens_to_keep=1):
thomwolf's avatar
thomwolf committed
967
968
    """ Filter a distribution of logits using top-k and/or nucleus (top-p) filtering
        Args:
thomwolf's avatar
thomwolf committed
969
            logits: logits distribution shape (batch size, vocabulary size)
970
971
            if top_k > 0: keep only top k tokens with highest probability (top-k filtering).
            if top_p < 1.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).
thomwolf's avatar
thomwolf committed
972
                Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)
thomwolf's avatar
thomwolf committed
973
            Make sure we keep at least min_tokens_to_keep per batch example in the output
thomwolf's avatar
thomwolf committed
974
975
976
        From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317
    """
    if top_k > 0:
thomwolf's avatar
thomwolf committed
977
        top_k = min(max(top_k, min_tokens_to_keep), logits.size(-1))  # Safety check
thomwolf's avatar
thomwolf committed
978
979
980
981
        # Remove all tokens with a probability less than the last token of the top-k
        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
        logits[indices_to_remove] = filter_value

982
    if top_p < 1.0:
thomwolf's avatar
thomwolf committed
983
984
985
        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)

thomwolf's avatar
thomwolf committed
986
        # Remove tokens with cumulative probability above the threshold (token with 0 are kept)
thomwolf's avatar
thomwolf committed
987
        sorted_indices_to_remove = cumulative_probs > top_p
thomwolf's avatar
thomwolf committed
988
989
990
        if min_tokens_to_keep > 1:
            # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below)
            sorted_indices_to_remove[..., :min_tokens_to_keep] = 0
thomwolf's avatar
thomwolf committed
991
992
993
994
995
        # Shift the indices to the right to keep also the first token above the threshold
        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
        sorted_indices_to_remove[..., 0] = 0

        # scatter sorted tensors to original indexing
996
        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
thomwolf's avatar
thomwolf committed
997
998
        logits[indices_to_remove] = filter_value
    return logits
thomwolf's avatar
thomwolf committed
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017


class BeamHypotheses(object):
    def __init__(self, n_hyp, max_length, length_penalty, early_stopping):
        """
        Initialize n-best list of hypotheses.
        """
        self.max_length = max_length - 1  # ignoring bos_token
        self.length_penalty = length_penalty
        self.early_stopping = early_stopping
        self.n_hyp = n_hyp
        self.hyp = []
        self.worst_score = 1e9

    def __len__(self):
        """
        Number of hypotheses in the list.
        """
        return len(self.hyp)
thomwolf's avatar
thomwolf committed
1018

thomwolf's avatar
thomwolf committed
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
    def add(self, hyp, sum_logprobs):
        """
        Add a new hypothesis to the list.
        """
        score = sum_logprobs / len(hyp) ** self.length_penalty
        if len(self) < self.n_hyp or score > self.worst_score:
            self.hyp.append((score, hyp))
            if len(self) > self.n_hyp:
                sorted_scores = sorted([(s, idx) for idx, (s, _) in enumerate(self.hyp)])
                del self.hyp[sorted_scores[0][1]]
                self.worst_score = sorted_scores[1][0]
            else:
                self.worst_score = min(score, self.worst_score)
thomwolf's avatar
thomwolf committed
1032

thomwolf's avatar
thomwolf committed
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
    def is_done(self, best_sum_logprobs):
        """
        If there are enough hypotheses and that none of the hypotheses being generated
        can become better than the worst one in the heap, then we are done with this sentence.
        """
        if len(self) < self.n_hyp:
            return False
        elif self.early_stopping:
            return True
        else:
            return self.worst_score >= best_sum_logprobs / self.max_length ** self.length_penalty
thomwolf's avatar
thomwolf committed
1044
1045


thomwolf's avatar
thomwolf committed
1046
1047
class Conv1D(nn.Module):
    def __init__(self, nf, nx):
thomwolf's avatar
thomwolf committed
1048
        """ Conv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2)
thomwolf's avatar
thomwolf committed
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
            Basically works like a Linear layer but the weights are transposed
        """
        super(Conv1D, self).__init__()
        self.nf = nf
        w = torch.empty(nx, nf)
        nn.init.normal_(w, std=0.02)
        self.weight = nn.Parameter(w)
        self.bias = nn.Parameter(torch.zeros(nf))

    def forward(self, x):
        size_out = x.size()[:-1] + (self.nf,)
        x = torch.addmm(self.bias, x.view(-1, x.size(-1)), self.weight)
        x = x.view(*size_out)
        return x


thomwolf's avatar
thomwolf committed
1065
1066
class PoolerStartLogits(nn.Module):
    """ Compute SQuAD start_logits from sequence hidden states. """
1067

thomwolf's avatar
thomwolf committed
1068
    def __init__(self, config):
thomwolf's avatar
thomwolf committed
1069
1070
1071
1072
1073
        super(PoolerStartLogits, self).__init__()
        self.dense = nn.Linear(config.hidden_size, 1)

    def forward(self, hidden_states, p_mask=None):
        """ Args:
1074
1075
1076
            **p_mask**: (`optional`) ``torch.FloatTensor`` of shape `(batch_size, seq_len)`
                invalid position mask such as query and special symbols (PAD, SEP, CLS)
                1.0 means token should be masked.
thomwolf's avatar
thomwolf committed
1077
        """
thomwolf's avatar
thomwolf committed
1078
1079
1080
        x = self.dense(hidden_states).squeeze(-1)

        if p_mask is not None:
1081
1082
1083
1084
            if next(self.parameters()).dtype == torch.float16:
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
1085
1086
1087
1088
1089
1090
1091

        return x


class PoolerEndLogits(nn.Module):
    """ Compute SQuAD end_logits from sequence hidden states and start token hidden state.
    """
1092

thomwolf's avatar
thomwolf committed
1093
1094
1095
1096
1097
1098
1099
1100
1101
    def __init__(self, config):
        super(PoolerEndLogits, self).__init__()
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dense_1 = nn.Linear(config.hidden_size, 1)

    def forward(self, hidden_states, start_states=None, start_positions=None, p_mask=None):
        """ Args:
1102
1103
1104
1105
1106
1107
            One of ``start_states``, ``start_positions`` should be not None.
            If both are set, ``start_positions`` overrides ``start_states``.

            **start_states**: ``torch.LongTensor`` of shape identical to hidden_states
                hidden states of the first tokens for the labeled span.
            **start_positions**: ``torch.LongTensor`` of shape ``(batch_size,)``
1108
                position of the first token for the labeled span:
1109
1110
1111
            **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, seq_len)``
                Mask of invalid position such as query and special symbols (PAD, SEP, CLS)
                1.0 means token should be masked.
thomwolf's avatar
thomwolf committed
1112
        """
1113
1114
1115
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
1116
        if start_positions is not None:
1117
            slen, hsz = hidden_states.shape[-2:]
1118
1119
1120
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions)  # shape (bsz, 1, hsz)
            start_states = start_states.expand(-1, slen, -1)  # shape (bsz, slen, hsz)
thomwolf's avatar
thomwolf committed
1121
1122
1123
1124
1125
1126
1127

        x = self.dense_0(torch.cat([hidden_states, start_states], dim=-1))
        x = self.activation(x)
        x = self.LayerNorm(x)
        x = self.dense_1(x).squeeze(-1)

        if p_mask is not None:
1128
1129
1130
1131
            if next(self.parameters()).dtype == torch.float16:
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
1132
1133
1134
1135
1136
1137

        return x


class PoolerAnswerClass(nn.Module):
    """ Compute SQuAD 2.0 answer class from classification and start tokens hidden states. """
1138

thomwolf's avatar
thomwolf committed
1139
1140
1141
1142
1143
1144
1145
    def __init__(self, config):
        super(PoolerAnswerClass, self).__init__()
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.dense_1 = nn.Linear(config.hidden_size, 1, bias=False)

    def forward(self, hidden_states, start_states=None, start_positions=None, cls_index=None):
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
        """
        Args:
            One of ``start_states``, ``start_positions`` should be not None.
            If both are set, ``start_positions`` overrides ``start_states``.

            **start_states**: ``torch.LongTensor`` of shape identical to ``hidden_states``.
                hidden states of the first tokens for the labeled span.
            **start_positions**: ``torch.LongTensor`` of shape ``(batch_size,)``
                position of the first token for the labeled span.
            **cls_index**: torch.LongTensor of shape ``(batch_size,)``
                position of the CLS token. If None, take the last token.

            note(Original repo):
                no dependency on end_feature so that we can obtain one single `cls_logits`
                for each sample
thomwolf's avatar
thomwolf committed
1161
        """
1162
        hsz = hidden_states.shape[-1]
1163
1164
1165
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
1166
        if start_positions is not None:
1167
1168
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
1169
1170

        if cls_index is not None:
1171
1172
            cls_index = cls_index[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            cls_token_state = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
1173
        else:
1174
            cls_token_state = hidden_states[:, -1, :]  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
1175
1176
1177
1178
1179
1180
1181
1182
1183

        x = self.dense_0(torch.cat([start_states, cls_token_state], dim=-1))
        x = self.activation(x)
        x = self.dense_1(x).squeeze(-1)

        return x


class SQuADHead(nn.Module):
1184
1185
1186
    r""" A SQuAD head inspired by XLNet.

    Parameters:
1187
        config (:class:`~transformers.XLNetConfig`): Model configuration class with all the parameters of the model.
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206

    Inputs:
        **hidden_states**: ``torch.FloatTensor`` of shape ``(batch_size, seq_len, hidden_size)``
            hidden states of sequence tokens
        **start_positions**: ``torch.LongTensor`` of shape ``(batch_size,)``
            position of the first token for the labeled span.
        **end_positions**: ``torch.LongTensor`` of shape ``(batch_size,)``
            position of the last token for the labeled span.
        **cls_index**: torch.LongTensor of shape ``(batch_size,)``
            position of the CLS token. If None, take the last token.
        **is_impossible**: ``torch.LongTensor`` of shape ``(batch_size,)``
            Whether the question has a possible answer in the paragraph or not.
        **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, seq_len)``
            Mask of invalid position such as query and special symbols (PAD, SEP, CLS)
            1.0 means token should be masked.

    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
        **loss**: (`optional`, returned if both ``start_positions`` and ``end_positions`` are provided) ``torch.FloatTensor`` of shape ``(1,)``:
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses.
thomwolf's avatar
thomwolf committed
1207
        **start_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided)
1208
1209
            ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top)``
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
thomwolf's avatar
thomwolf committed
1210
        **start_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided)
1211
1212
            ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top)``
            Indices for the top config.start_n_top start token possibilities (beam-search).
thomwolf's avatar
thomwolf committed
1213
        **end_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided)
1214
1215
            ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)``
            Log probabilities for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search).
thomwolf's avatar
thomwolf committed
1216
        **end_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided)
1217
1218
            ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)``
            Indices for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search).
thomwolf's avatar
thomwolf committed
1219
        **cls_logits**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided)
1220
1221
            ``torch.FloatTensor`` of shape ``(batch_size,)``
            Log probabilities for the ``is_impossible`` label of the answers.
thomwolf's avatar
thomwolf committed
1222
    """
1223

thomwolf's avatar
thomwolf committed
1224
1225
1226
1227
1228
1229
1230
1231
1232
    def __init__(self, config):
        super(SQuADHead, self).__init__()
        self.start_n_top = config.start_n_top
        self.end_n_top = config.end_n_top

        self.start_logits = PoolerStartLogits(config)
        self.end_logits = PoolerEndLogits(config)
        self.answer_class = PoolerAnswerClass(config)

1233
1234
1235
    def forward(
        self, hidden_states, start_positions=None, end_positions=None, cls_index=None, is_impossible=None, p_mask=None
    ):
thomwolf's avatar
thomwolf committed
1236
1237
        outputs = ()

thomwolf's avatar
thomwolf committed
1238
        start_logits = self.start_logits(hidden_states, p_mask=p_mask)
thomwolf's avatar
thomwolf committed
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261

        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, let's remove the dimension added by batch splitting
            for x in (start_positions, end_positions, cls_index, is_impossible):
                if x is not None and x.dim() > 1:
                    x.squeeze_(-1)

            # during training, compute the end logits based on the ground truth of the start position
            end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask)

            loss_fct = CrossEntropyLoss()
            start_loss = loss_fct(start_logits, start_positions)
            end_loss = loss_fct(end_logits, end_positions)
            total_loss = (start_loss + end_loss) / 2

            if cls_index is not None and is_impossible is not None:
                # Predict answerability from the representation of CLS and START
                cls_logits = self.answer_class(hidden_states, start_positions=start_positions, cls_index=cls_index)
                loss_fct_cls = nn.BCEWithLogitsLoss()
                cls_loss = loss_fct_cls(cls_logits, is_impossible)

                # note(zhiliny): by default multiply the loss by 0.5 so that the scale is comparable to start_loss and end_loss
                total_loss += cls_loss * 0.5
1262
1263

            outputs = (total_loss,) + outputs
thomwolf's avatar
thomwolf committed
1264
1265
1266
1267

        else:
            # during inference, compute the end logits based on beam search
            bsz, slen, hsz = hidden_states.size()
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
            start_log_probs = F.softmax(start_logits, dim=-1)  # shape (bsz, slen)

            start_top_log_probs, start_top_index = torch.topk(
                start_log_probs, self.start_n_top, dim=-1
            )  # shape (bsz, start_n_top)
            start_top_index_exp = start_top_index.unsqueeze(-1).expand(-1, -1, hsz)  # shape (bsz, start_n_top, hsz)
            start_states = torch.gather(hidden_states, -2, start_top_index_exp)  # shape (bsz, start_n_top, hsz)
            start_states = start_states.unsqueeze(1).expand(-1, slen, -1, -1)  # shape (bsz, slen, start_n_top, hsz)

            hidden_states_expanded = hidden_states.unsqueeze(2).expand_as(
                start_states
            )  # shape (bsz, slen, start_n_top, hsz)
thomwolf's avatar
thomwolf committed
1280
1281
            p_mask = p_mask.unsqueeze(-1) if p_mask is not None else None
            end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask)
1282
            end_log_probs = F.softmax(end_logits, dim=1)  # shape (bsz, slen, start_n_top)
thomwolf's avatar
thomwolf committed
1283

1284
1285
1286
            end_top_log_probs, end_top_index = torch.topk(
                end_log_probs, self.end_n_top, dim=1
            )  # shape (bsz, end_n_top, start_n_top)
thomwolf's avatar
thomwolf committed
1287
1288
1289
1290
1291
1292
1293
1294
1295
            end_top_log_probs = end_top_log_probs.view(-1, self.start_n_top * self.end_n_top)
            end_top_index = end_top_index.view(-1, self.start_n_top * self.end_n_top)

            start_states = torch.einsum("blh,bl->bh", hidden_states, start_log_probs)
            cls_logits = self.answer_class(hidden_states, start_states=start_states, cls_index=cls_index)

            outputs = (start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits) + outputs

        # return start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits
1296
        # or (if labels are provided) (total_loss,)
thomwolf's avatar
thomwolf committed
1297
1298
1299
1300
        return outputs


class SequenceSummary(nn.Module):
thomwolf's avatar
thomwolf committed
1301
    r""" Compute a single vector summary of a sequence hidden states according to various possibilities:
thomwolf's avatar
thomwolf committed
1302
1303
1304
1305
1306
        Args of the config class:
            summary_type:
                - 'last' => [default] take the last token hidden state (like XLNet)
                - 'first' => take the first token hidden state (like Bert)
                - 'mean' => take the mean of all tokens hidden states
thomwolf's avatar
thomwolf committed
1307
                - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
thomwolf's avatar
thomwolf committed
1308
1309
                - 'attn' => Not implemented now, use multi-head attention
            summary_use_proj: Add a projection after the vector extraction
1310
            summary_proj_to_labels: If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False.
1311
            summary_activation: 'tanh' => add a tanh activation to the output, Other => no activation. Default
1312
1313
            summary_first_dropout: Add a dropout before the projection and activation
            summary_last_dropout: Add a dropout after the projection and activation
thomwolf's avatar
thomwolf committed
1314
    """
1315

thomwolf's avatar
thomwolf committed
1316
    def __init__(self, config):
thomwolf's avatar
thomwolf committed
1317
1318
        super(SequenceSummary, self).__init__()

1319
1320
        self.summary_type = config.summary_type if hasattr(config, "summary_type") else "last"
        if self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
1321
1322
1323
1324
1325
            # We should use a standard multi-head attention module with absolute positional embedding for that.
            # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276
            # We can probably just use the multi-head attention module of PyTorch >=1.1.0
            raise NotImplementedError

thomwolf's avatar
thomwolf committed
1326
        self.summary = Identity()
1327
1328
        if hasattr(config, "summary_use_proj") and config.summary_use_proj:
            if hasattr(config, "summary_proj_to_labels") and config.summary_proj_to_labels and config.num_labels > 0:
1329
                num_classes = config.num_labels
thomwolf's avatar
thomwolf committed
1330
1331
1332
1333
            else:
                num_classes = config.hidden_size
            self.summary = nn.Linear(config.hidden_size, num_classes)

thomwolf's avatar
thomwolf committed
1334
        self.activation = Identity()
1335
        if hasattr(config, "summary_activation") and config.summary_activation == "tanh":
thomwolf's avatar
thomwolf committed
1336
1337
            self.activation = nn.Tanh()

thomwolf's avatar
thomwolf committed
1338
        self.first_dropout = Identity()
1339
        if hasattr(config, "summary_first_dropout") and config.summary_first_dropout > 0:
1340
1341
            self.first_dropout = nn.Dropout(config.summary_first_dropout)

thomwolf's avatar
thomwolf committed
1342
        self.last_dropout = Identity()
1343
        if hasattr(config, "summary_last_dropout") and config.summary_last_dropout > 0:
1344
            self.last_dropout = nn.Dropout(config.summary_last_dropout)
thomwolf's avatar
thomwolf committed
1345

thomwolf's avatar
thomwolf committed
1346
    def forward(self, hidden_states, cls_index=None):
1347
        """ hidden_states: float Tensor in shape [bsz, ..., seq_len, hidden_size], the hidden-states of the last layer.
thomwolf's avatar
thomwolf committed
1348
            cls_index: [optional] position of the classification token if summary_type == 'cls_index',
thomwolf's avatar
thomwolf committed
1349
                shape (bsz,) or more generally (bsz, ...) where ... are optional leading dimensions of hidden_states.
thomwolf's avatar
thomwolf committed
1350
                if summary_type == 'cls_index' and cls_index is None:
thomwolf's avatar
thomwolf committed
1351
1352
                    we take the last token of the sequence as classification token
        """
1353
        if self.summary_type == "last":
thomwolf's avatar
thomwolf committed
1354
            output = hidden_states[:, -1]
1355
        elif self.summary_type == "first":
thomwolf's avatar
thomwolf committed
1356
            output = hidden_states[:, 0]
1357
        elif self.summary_type == "mean":
thomwolf's avatar
thomwolf committed
1358
            output = hidden_states.mean(dim=1)
1359
        elif self.summary_type == "cls_index":
thomwolf's avatar
thomwolf committed
1360
            if cls_index is None:
1361
                cls_index = torch.full_like(hidden_states[..., :1, :], hidden_states.shape[-2] - 1, dtype=torch.long)
thomwolf's avatar
thomwolf committed
1362
            else:
thomwolf's avatar
thomwolf committed
1363
                cls_index = cls_index.unsqueeze(-1).unsqueeze(-1)
1364
                cls_index = cls_index.expand((-1,) * (cls_index.dim() - 1) + (hidden_states.size(-1),))
thomwolf's avatar
thomwolf committed
1365
            # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states
1366
1367
            output = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, XX, hidden_size)
        elif self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
1368
1369
            raise NotImplementedError

1370
        output = self.first_dropout(output)
thomwolf's avatar
thomwolf committed
1371
1372
        output = self.summary(output)
        output = self.activation(output)
1373
        output = self.last_dropout(output)
thomwolf's avatar
thomwolf committed
1374
1375
1376
1377

        return output


1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
def prune_linear_layer(layer, index, dim=0):
    """ Prune a linear layer (a model parameters) to keep only entries in index.
        Return the pruned layer as a new layer with requires_grad=True.
        Used to remove heads.
    """
    index = index.to(layer.weight.device)
    W = layer.weight.index_select(dim, index).clone().detach()
    if layer.bias is not None:
        if dim == 1:
            b = layer.bias.clone().detach()
        else:
            b = layer.bias[index].clone().detach()
    new_size = list(layer.weight.size())
    new_size[dim] = len(index)
    new_layer = nn.Linear(new_size[1], new_size[0], bias=layer.bias is not None).to(layer.weight.device)
    new_layer.weight.requires_grad = False
    new_layer.weight.copy_(W.contiguous())
    new_layer.weight.requires_grad = True
    if layer.bias is not None:
        new_layer.bias.requires_grad = False
        new_layer.bias.copy_(b.contiguous())
        new_layer.bias.requires_grad = True
    return new_layer


def prune_conv1d_layer(layer, index, dim=1):
    """ Prune a Conv1D layer (a model parameters) to keep only entries in index.
        A Conv1D work as a Linear layer (see e.g. BERT) but the weights are transposed.
        Return the pruned layer as a new layer with requires_grad=True.
        Used to remove heads.
    """
    index = index.to(layer.weight.device)
    W = layer.weight.index_select(dim, index).clone().detach()
    if dim == 0:
        b = layer.bias.clone().detach()
    else:
        b = layer.bias[index].clone().detach()
    new_size = list(layer.weight.size())
    new_size[dim] = len(index)
    new_layer = Conv1D(new_size[1], new_size[0]).to(layer.weight.device)
    new_layer.weight.requires_grad = False
    new_layer.weight.copy_(W.contiguous())
    new_layer.weight.requires_grad = True
    new_layer.bias.requires_grad = False
    new_layer.bias.copy_(b.contiguous())
    new_layer.bias.requires_grad = True
    return new_layer
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437


def prune_layer(layer, index, dim=None):
    """ Prune a Conv1D or nn.Linear layer (a model parameters) to keep only entries in index.
        Return the pruned layer as a new layer with requires_grad=True.
        Used to remove heads.
    """
    if isinstance(layer, nn.Linear):
        return prune_linear_layer(layer, index, dim=0 if dim is None else dim)
    elif isinstance(layer, Conv1D):
        return prune_conv1d_layer(layer, index, dim=1 if dim is None else dim)
    else:
        raise ValueError("Can't prune layer of class {}".format(layer.__class__))