"docs/vscode:/vscode.git/clone" did not exist on "b8f1cde931392551f74a9abef5d2724c3cbc2208"
modeling_utils.py 202 KB
Newer Older
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team.
3
4
5
6
7
8
9
10
11
12
13
14
15
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
16
import collections
17
import gc
18
import importlib.metadata
Yih-Dar's avatar
Yih-Dar committed
19
import inspect
Sylvain Gugger's avatar
Sylvain Gugger committed
20
import json
21
import os
22
import re
Sylvain Gugger's avatar
Sylvain Gugger committed
23
24
import shutil
import tempfile
25
import warnings
26
from contextlib import contextmanager
27
from dataclasses import dataclass
28
from functools import partial, wraps
29
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
30
31

import torch
32
from packaging import version
Sylvain Gugger's avatar
Sylvain Gugger committed
33
from torch import Tensor, nn
34
from torch.nn import CrossEntropyLoss
35

36
from .activations import get_activation
37
from .configuration_utils import PretrainedConfig
38
from .deepspeed import deepspeed_config, is_deepspeed_zero3_enabled
39
from .dynamic_module_utils import custom_object_save
40
from .generation import GenerationConfig, GenerationMixin
41
from .lib_integrations import PeftAdapterMixin
42
43
44
45
from .pytorch_utils import (  # noqa: F401
    Conv1D,
    apply_chunking_to_forward,
    find_pruneable_heads_and_indices,
Thomas Wang's avatar
Thomas Wang committed
46
    id_tensor_storage,
47
48
49
50
    prune_conv1d_layer,
    prune_layer,
    prune_linear_layer,
)
51
from .utils import (
52
53
    ADAPTER_SAFE_WEIGHTS_NAME,
    ADAPTER_WEIGHTS_NAME,
54
    CONFIG_NAME,
Aymeric Augustin's avatar
Aymeric Augustin committed
55
    DUMMY_INPUTS,
56
    FLAX_WEIGHTS_NAME,
57
58
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
59
60
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
Sylvain Gugger's avatar
Sylvain Gugger committed
61
    WEIGHTS_INDEX_NAME,
62
    WEIGHTS_NAME,
63
    ContextManagers,
64
    ModelOutput,
Sylvain Gugger's avatar
Sylvain Gugger committed
65
    PushToHubMixin,
66
    cached_file,
67
    copy_func,
68
    download_url,
69
    extract_commit_hash,
70
    has_file,
71
    is_accelerate_available,
Marc Sun's avatar
Marc Sun committed
72
    is_auto_gptq_available,
73
    is_bitsandbytes_available,
74
    is_offline_mode,
75
    is_optimum_available,
76
    is_peft_available,
77
    is_remote_url,
78
    is_safetensors_available,
79
    is_torch_tpu_available,
80
    logging,
Sylvain Gugger's avatar
Sylvain Gugger committed
81
    replace_return_docstrings,
82
    strtobool,
83
)
84
from .utils.hub import convert_file_size_to_int, get_checkpoint_shard_files
85
from .utils.import_utils import ENV_VARS_TRUE_VALUES, is_sagemaker_mp_enabled, is_torch_fx_proxy
Marc Sun's avatar
Marc Sun committed
86
from .utils.quantization_config import BitsAndBytesConfig, GPTQConfig, QuantizationMethod
87
from .utils.versions import require_version_core
88

Aymeric Augustin's avatar
Aymeric Augustin committed
89

90
91
92
XLA_USE_BF16 = os.environ.get("XLA_USE_BF16", "0").upper()
XLA_DOWNCAST_BF16 = os.environ.get("XLA_DOWNCAST_BF16", "0").upper()

93
94
if is_accelerate_available():
    from accelerate import dispatch_model, infer_auto_device_map, init_empty_weights
95
    from accelerate.hooks import add_hook_to_module
96
    from accelerate.utils import (
97
        check_tied_parameters_on_same_device,
98
        find_tied_parameters,
99
        get_balanced_memory,
100
101
102
103
104
105
        load_offloaded_weights,
        offload_weight,
        save_offload_index,
        set_module_tensor_to_device,
    )

106
107
108
109
if is_safetensors_available():
    from safetensors import safe_open
    from safetensors.torch import load_file as safe_load_file
    from safetensors.torch import save_file as safe_save_file
110

Lysandre Debut's avatar
Lysandre Debut committed
111
logger = logging.get_logger(__name__)
112

113
114
115
116

_init_weights = True


117
def is_fsdp_enabled():
118
    return torch.distributed.is_initialized() and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
119
120
121


def is_fsdp_enabled_and_dist_rank_0():
122
    return is_fsdp_enabled() and torch.distributed.get_rank() == 0
123
124


125
126
127
128
129
130
131
132
if is_sagemaker_mp_enabled():
    import smdistributed.modelparallel.torch as smp
    from smdistributed.modelparallel import __version__ as SMP_VERSION

    IS_SAGEMAKER_MP_POST_1_10 = version.parse(SMP_VERSION) >= version.parse("1.10")
else:
    IS_SAGEMAKER_MP_POST_1_10 = False

133
134
135
if is_peft_available():
    from .utils import find_adapter_config_file

136

137
138
139
140
141
142
143
144
@contextmanager
def no_init_weights(_enable=True):
    """
    Context manager to globally disable weight initialization to speed up loading large models.

    TODO(Patrick): Delete safety argument `_enable=True` at next major version. .
    """
    global _init_weights
145
    old_init_weights = _init_weights
146
147
148
149
150
    if _enable:
        _init_weights = False
    try:
        yield
    finally:
151
        _init_weights = old_init_weights
152
153


thomwolf's avatar
thomwolf committed
154
155
156
157
158
try:
    from torch.nn import Identity
except ImportError:
    # Older PyTorch compatibility
    class Identity(nn.Module):
Lysandre's avatar
Lysandre committed
159
        r"""A placeholder identity operator that is argument-insensitive."""
160

thomwolf's avatar
thomwolf committed
161
        def __init__(self, *args, **kwargs):
Julien Chaumond's avatar
Julien Chaumond committed
162
            super().__init__()
thomwolf's avatar
thomwolf committed
163
164
165
166

        def forward(self, input):
            return input

167

Lysandre Debut's avatar
Lysandre Debut committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
def get_parameter_device(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    try:
        return next(parameter.parameters()).device
    except StopIteration:
        # For nn.DataParallel compatibility in PyTorch 1.5

        def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
            tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
            return tuples

        gen = parameter._named_members(get_members_fn=find_tensor_attributes)
        first_tuple = next(gen)
        return first_tuple[1].device


183
184
185
186
def get_first_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    """
    Returns the first parameter dtype (can be non-floating) or asserts if none were found.
    """
Lysandre Debut's avatar
Lysandre Debut committed
187
188
189
    try:
        return next(parameter.parameters()).dtype
    except StopIteration:
Sylvain Gugger's avatar
Sylvain Gugger committed
190
        # For nn.DataParallel compatibility in PyTorch > 1.5
Lysandre Debut's avatar
Lysandre Debut committed
191
192
193
194
195
196
197
198
199
200

        def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
            tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
            return tuples

        gen = parameter._named_members(get_members_fn=find_tensor_attributes)
        first_tuple = next(gen)
        return first_tuple[1].dtype


201
202
def get_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
203
    Returns the first found floating dtype in parameters if there is one, otherwise returns the last dtype it found.
204
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
205
206
207
208
    last_dtype = None
    for t in parameter.parameters():
        last_dtype = t.dtype
        if t.is_floating_point():
209
210
211
            # Adding fix for https://github.com/pytorch/xla/issues/4152
            # Fixes issue where the model code passes a value that is out of range for XLA_USE_BF16=1
            # and XLA_DOWNCAST_BF16=1 so the conversion would cast it to -inf
212
213
214
215
216
            # NOTE: `is_torch_tpu_available()` is checked last as it induces a graph break in torch dynamo
            if XLA_USE_BF16 in ENV_VARS_TRUE_VALUES and is_torch_tpu_available():
                return torch.bfloat16
            if XLA_DOWNCAST_BF16 in ENV_VARS_TRUE_VALUES and is_torch_tpu_available():
                if t.dtype == torch.float:
217
                    return torch.bfloat16
218
219
                if t.dtype == torch.double:
                    return torch.float32
Sylvain Gugger's avatar
Sylvain Gugger committed
220
            return t.dtype
221

Sylvain Gugger's avatar
Sylvain Gugger committed
222
223
224
    if last_dtype is not None:
        # if no floating dtype was found return whatever the first dtype is
        return last_dtype
225

226
227
228
229
230
231
232
233
234
235
236
237
238
    # For nn.DataParallel compatibility in PyTorch > 1.5
    def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
        tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
        return tuples

    gen = parameter._named_members(get_members_fn=find_tensor_attributes)
    last_tuple = None
    for tuple in gen:
        last_tuple = tuple
        if tuple[1].is_floating_point():
            return tuple[1].dtype

    if last_tuple is not None:
239
240
        # fallback to the last dtype
        return last_tuple[1].dtype
241

242
243
244
245
246
247
248
    # fallback to buffer dtype
    for t in parameter.buffers():
        last_dtype = t.dtype
        if t.is_floating_point():
            return t.dtype
    return last_dtype

249
250
251
252
253
254
255
256
257
258
259
260
261
262

def get_state_dict_float_dtype(state_dict):
    """
    Returns the first found floating dtype in `state_dict` or asserts if none were found.
    """
    for t in state_dict.values():
        if t.is_floating_point():
            return t.dtype

    raise ValueError("couldn't find any floating point dtypes in state_dict")


def get_state_dict_dtype(state_dict):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
263
    Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype.
264
265
266
267
268
269
270
    """
    for t in state_dict.values():
        if t.is_floating_point():
            return t.dtype

    # if no floating dtype was found return whatever the first dtype is
    else:
Sylvain Gugger's avatar
Sylvain Gugger committed
271
        return next(state_dict.values()).dtype
272
273


Sylvain Gugger's avatar
Sylvain Gugger committed
274
275
276
277
278
279
280
281
282
283
284
285
286
def dtype_byte_size(dtype):
    """
    Returns the size (in bytes) occupied by one parameter of type `dtype`.

    Example:

    ```py
    >>> dtype_byte_size(torch.float32)
    4
    ```
    """
    if dtype == torch.bool:
        return 1 / 8
287
    bit_search = re.search(r"[^\d](\d+)$", str(dtype))
Sylvain Gugger's avatar
Sylvain Gugger committed
288
289
290
291
292
293
    if bit_search is None:
        raise ValueError(f"`dtype` is not a valid dtype: {dtype}.")
    bit_size = int(bit_search.groups()[0])
    return bit_size // 8


294
295
296
def shard_checkpoint(
    state_dict: Dict[str, torch.Tensor], max_shard_size: Union[int, str] = "10GB", weights_name: str = WEIGHTS_NAME
):
Sylvain Gugger's avatar
Sylvain Gugger committed
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
    """
    Splits a model state dictionary in sub-checkpoints so that the final size of each sub-checkpoint does not exceed a
    given size.

    The sub-checkpoints are determined by iterating through the `state_dict` in the order of its keys, so there is no
    optimization made to make each sub-checkpoint as close as possible to the maximum size passed. For example, if the
    limit is 10GB and we have weights of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB],
    [6+2+2GB] and not [6+2+2GB], [6+2GB], [6GB].

    <Tip warning={true}>

    If one of the model's weight is bigger that `max_sahrd_size`, it will end up in its own sub-checkpoint which will
    have a size greater than `max_shard_size`.

    </Tip>

    Args:
        state_dict (`Dict[str, torch.Tensor]`): The state dictionary of a model to save.
        max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`):
            The maximum size of each sub-checkpoint. If expressed as a string, needs to be digits followed by a unit
            (like `"5MB"`).
318
319
        weights_name (`str`, *optional*, defaults to `"pytorch_model.bin"`):
            The name of the model save file.
Sylvain Gugger's avatar
Sylvain Gugger committed
320
321
322
    """
    max_shard_size = convert_file_size_to_int(max_shard_size)

Thomas Wang's avatar
Thomas Wang committed
323
324
    sharded_state_dicts = [{}]
    last_block_size = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
325
    total_size = 0
Thomas Wang's avatar
Thomas Wang committed
326
    storage_id_to_block = {}
Sylvain Gugger's avatar
Sylvain Gugger committed
327
328

    for key, weight in state_dict.items():
329
330
331
332
333
334
        # when bnb serialization is used the weights in the state dict can be strings
        # check: https://github.com/huggingface/transformers/pull/24416 for more details
        if isinstance(weight, str):
            continue
        else:
            storage_id = id_tensor_storage(weight)
Thomas Wang's avatar
Thomas Wang committed
335
336
337
338
339
340
341

        # If a `weight` shares the same underlying storage as another tensor, we put `weight` in the same `block`
        if storage_id in storage_id_to_block:
            block_id = storage_id_to_block[storage_id]
            sharded_state_dicts[block_id][key] = weight
            continue

Sylvain Gugger's avatar
Sylvain Gugger committed
342
343
        weight_size = weight.numel() * dtype_byte_size(weight.dtype)

Sylvain Gugger's avatar
Sylvain Gugger committed
344
345
346
        # If this weight is going to tip up over the maximal size, we split, but only if we have put at least one
        # weight in the current shard.
        if last_block_size + weight_size > max_shard_size and len(sharded_state_dicts[-1]) > 0:
Thomas Wang's avatar
Thomas Wang committed
347
348
            sharded_state_dicts.append({})
            last_block_size = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
349

Thomas Wang's avatar
Thomas Wang committed
350
351
        sharded_state_dicts[-1][key] = weight
        last_block_size += weight_size
Sylvain Gugger's avatar
Sylvain Gugger committed
352
        total_size += weight_size
Thomas Wang's avatar
Thomas Wang committed
353
        storage_id_to_block[storage_id] = len(sharded_state_dicts) - 1
Sylvain Gugger's avatar
Sylvain Gugger committed
354
355
356

    # If we only have one shard, we return it
    if len(sharded_state_dicts) == 1:
357
        return {weights_name: sharded_state_dicts[0]}, None
Sylvain Gugger's avatar
Sylvain Gugger committed
358
359
360
361
362

    # Otherwise, let's build the index
    weight_map = {}
    shards = {}
    for idx, shard in enumerate(sharded_state_dicts):
363
364
365
366
        shard_file = weights_name.replace(".bin", f"-{idx+1:05d}-of-{len(sharded_state_dicts):05d}.bin")
        shard_file = shard_file.replace(
            ".safetensors", f"-{idx + 1:05d}-of-{len(sharded_state_dicts):05d}.safetensors"
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
367
368
369
370
371
372
373
374
375
376
        shards[shard_file] = shard
        for key in shard.keys():
            weight_map[key] = shard_file

    # Add the metadata
    metadata = {"total_size": total_size}
    index = {"metadata": metadata, "weight_map": weight_map}
    return shards, index


377
def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True):
378
379
380
381
382
383
384
385
386
387
388
389
390
    """
    This is the same as
    [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict)
    but for a sharded checkpoint.

    This load is performed efficiently: each checkpoint shard is loaded one by one in RAM and deleted after being
    loaded in the model.

    Args:
        model (`torch.nn.Module`): The model in which to load the checkpoint.
        folder (`str` or `os.PathLike`): A path to a folder containing the sharded checkpoint.
        strict (`bool`, *optional`, defaults to `True`):
            Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint.
391
392
393
        prefer_safe (`bool`, *optional*, defaults to `False`)
            If both safetensors and PyTorch save files are present in checkpoint and `prefer_safe` is True, the
            safetensors files will be loaded. Otherwise, PyTorch files are always loaded when possible.
394
395
396
397
398
399
400
401

    Returns:
        `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields
            - `missing_keys` is a list of str containing the missing keys
            - `unexpected_keys` is a list of str containing the unexpected keys
    """
    # Load the index
    index_file = os.path.join(folder, WEIGHTS_INDEX_NAME)
402
    safe_index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME)
403

404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
    index_present = os.path.isfile(index_file)
    safe_index_present = os.path.isfile(safe_index_file)

    if not index_present and not (safe_index_present and is_safetensors_available()):
        filenames = (
            (WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_INDEX_NAME) if is_safetensors_available() else (WEIGHTS_INDEX_NAME,)
        )
        raise ValueError(f"Can't find a checkpoint index ({' or '.join(filenames)}) in {folder}.")

    load_safe = False
    if safe_index_present:
        if prefer_safe:
            if is_safetensors_available():
                load_safe = True  # load safe due to preference
            else:
                logger.warning(
                    f"Cannot load sharded checkpoint at {folder} safely since safetensors is not installed!"
                )
        elif not index_present:
            load_safe = True  # load safe since we have no other choice

    load_index = safe_index_file if load_safe else index_file

    with open(load_index, "r", encoding="utf-8") as f:
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
        index = json.load(f)

    shard_files = list(set(index["weight_map"].values()))

    # If strict=True, error before loading any of the state dicts.
    loaded_keys = index["weight_map"].keys()
    model_keys = model.state_dict().keys()
    missing_keys = [key for key in model_keys if key not in loaded_keys]
    unexpected_keys = [key for key in loaded_keys if key not in model_keys]
    if strict and (len(missing_keys) > 0 or len(unexpected_keys) > 0):
        error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}"
        if len(missing_keys) > 0:
            str_missing_keys = ",".join([f'"{k}"' for k in missing_keys])
            error_message += f"\nMissing key(s): {str_missing_keys}."
        if len(unexpected_keys) > 0:
            str_unexpected_keys = ",".join([f'"{k}"' for k in unexpected_keys])
            error_message += f"\nMissing key(s): {str_unexpected_keys}."
        raise RuntimeError(error_message)

447
448
    loader = safe_load_file if load_safe else partial(torch.load, map_location="cpu")

449
    for shard_file in shard_files:
450
        state_dict = loader(os.path.join(folder, shard_file))
451
452
        model.load_state_dict(state_dict, strict=False)

453
        # Make sure memory is freed before we load the next state dict.
454
455
456
457
458
459
460
        del state_dict
        gc.collect()

    # Return the same thing as PyTorch load_state_dict function.
    return torch.nn.modules.module._IncompatibleKeys(missing_keys, unexpected_keys)


Sylvain Gugger's avatar
Sylvain Gugger committed
461
462
463
464
def load_state_dict(checkpoint_file: Union[str, os.PathLike]):
    """
    Reads a PyTorch checkpoint file, returning properly formatted errors if they arise.
    """
465
466
467
468
469
470
471
472
473
474
475
476
477
478
    if checkpoint_file.endswith(".safetensors") and is_safetensors_available():
        # Check format of the archive
        with safe_open(checkpoint_file, framework="pt") as f:
            metadata = f.metadata()
        if metadata.get("format") not in ["pt", "tf", "flax"]:
            raise OSError(
                f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure "
                "you save your model with the `save_pretrained` method."
            )
        elif metadata["format"] != "pt":
            raise NotImplementedError(
                f"Conversion from a {metadata['format']} safetensors archive to PyTorch is not implemented yet."
            )
        return safe_load_file(checkpoint_file)
Sylvain Gugger's avatar
Sylvain Gugger committed
479
    try:
480
        if (
481
            (is_deepspeed_zero3_enabled() or is_fsdp_enabled())
482
483
484
            and torch.distributed.is_initialized()
            and torch.distributed.get_rank() > 0
        ):
485
486
487
488
            map_location = "meta"
        else:
            map_location = "cpu"
        return torch.load(checkpoint_file, map_location=map_location)
Sylvain Gugger's avatar
Sylvain Gugger committed
489
490
491
    except Exception as e:
        try:
            with open(checkpoint_file) as f:
492
                if f.read(7) == "version":
Sylvain Gugger's avatar
Sylvain Gugger committed
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
                    raise OSError(
                        "You seem to have cloned a repository without having git-lfs installed. Please install "
                        "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder "
                        "you cloned."
                    )
                else:
                    raise ValueError(
                        f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained "
                        "model. Make sure you have saved the model properly."
                    ) from e
        except (UnicodeDecodeError, ValueError):
            raise OSError(
                f"Unable to load weights from pytorch checkpoint file for '{checkpoint_file}' "
                f"at '{checkpoint_file}'. "
                "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True."
            )


511
512
513
514
515
516
517
518
519
520
521
def set_initialized_submodules(model, state_dict_keys):
    """
    Sets the `_is_hf_initialized` flag in all submodules of a given model when all its weights are in the loaded state
    dict.
    """
    for module_name, module in model.named_modules():
        loaded_keys = [k.replace(f"{module_name}.", "") for k in state_dict_keys if k.startswith(f"{module_name}.")]
        if len(set(module.state_dict().keys()) - set(loaded_keys)) == 0:
            module._is_hf_initialized = True


Sylvain Gugger's avatar
Sylvain Gugger committed
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
def _load_state_dict_into_model(model_to_load, state_dict, start_prefix):
    # Convert old format to new format if needed from a PyTorch state_dict
    old_keys = []
    new_keys = []
    for key in state_dict.keys():
        new_key = None
        if "gamma" in key:
            new_key = key.replace("gamma", "weight")
        if "beta" in key:
            new_key = key.replace("beta", "bias")
        if new_key:
            old_keys.append(key)
            new_keys.append(new_key)
    for old_key, new_key in zip(old_keys, new_keys):
        state_dict[new_key] = state_dict.pop(old_key)

    # copy state_dict so _load_from_state_dict can modify it
    metadata = getattr(state_dict, "_metadata", None)
    state_dict = state_dict.copy()
    if metadata is not None:
        state_dict._metadata = metadata

    error_msgs = []

    # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants
    # so we need to apply the function recursively.
548
    def load(module: nn.Module, state_dict, prefix=""):
Sylvain Gugger's avatar
Sylvain Gugger committed
549
550
        local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
        args = (state_dict, prefix, local_metadata, True, [], [], error_msgs)
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
        # Parameters of module and children will start with prefix. We can exit early if there are none in this
        # state_dict
        if len([key for key in state_dict if key.startswith(prefix)]) > 0:
            if is_deepspeed_zero3_enabled():
                import deepspeed

                # In sharded models, each shard has only part of the full state_dict, so only gather
                # parameters that are in the current state_dict.
                named_parameters = dict(module.named_parameters(prefix=prefix[:-1], recurse=False))
                params_to_gather = [named_parameters[k] for k in state_dict.keys() if k in named_parameters]
                if len(params_to_gather) > 0:
                    # because zero3 puts placeholders in model params, this context
                    # manager gathers (unpartitions) the params of the current layer, then loads from
                    # the state dict and then re-partitions them again
                    with deepspeed.zero.GatheredParameters(params_to_gather, modifier_rank=0):
                        if torch.distributed.get_rank() == 0:
                            module._load_from_state_dict(*args)
            else:
                module._load_from_state_dict(*args)
Sylvain Gugger's avatar
Sylvain Gugger committed
570
571
572

        for name, child in module._modules.items():
            if child is not None:
573
                load(child, state_dict, prefix + name + ".")
Sylvain Gugger's avatar
Sylvain Gugger committed
574

575
576
577
578
    load(model_to_load, state_dict, prefix=start_prefix)
    # Delete `state_dict` so it could be collected by GC earlier. Note that `state_dict` is a copy of the argument, so
    # it's safe to delete it.
    del state_dict
Sylvain Gugger's avatar
Sylvain Gugger committed
579
580
581
582

    return error_msgs


583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
def find_submodule_and_param_name(model, long_key, start_prefix):
    """
    A helper util to find the last sub-module and the param/buffer name. If `start_prefix` is supplied it'll be removed
    from the start of the key
    """

    if len(start_prefix) > 0 and long_key.startswith(start_prefix):
        long_key = ".".join(long_key.split(".")[1:])

    split_key = long_key.split(".")
    submodule = model
    while len(split_key) > 1:
        if hasattr(submodule, split_key[0]):
            submodule = getattr(submodule, split_key[0])
            del split_key[0]
        else:
            submodule = None
            break
    if submodule == model:
        submodule = None
    return submodule, split_key[0]


def _move_model_to_meta(model, loaded_state_dict_keys, start_prefix):
    """
    Moves `loaded_state_dict_keys` in model to meta device which frees up the memory taken by those params.

    `start_prefix` is used for models which insert their name into model keys, e.g. `bert` in
    `bert.pooler.dense.weight`

    """

    # dematerialize param storage for keys that are going to be replaced by state_dict, by
    # putting those on the meta device
    for k in loaded_state_dict_keys:
        submodule, param_name = find_submodule_and_param_name(model, k, start_prefix)
        if submodule is not None:
            # selectively switch to the meta device only those params/buffers that will
            # be next replaced from state_dict. This a complex way to do p.to_("meta")
            # since we have no in-place to_ for tensors.
            new_val = getattr(submodule, param_name)
            if isinstance(new_val, torch.nn.Parameter):
                # isinstance returns False for Params on meta device, so switch after the check
                new_val = torch.nn.Parameter(new_val.to("meta"))
            else:
                new_val = new_val.to("meta")
            setattr(submodule, param_name, new_val)


632
633
634
635
636
637
638
639
640
641
642
643
def _load_state_dict_into_meta_model(
    model,
    state_dict,
    loaded_state_dict_keys,  # left for now but could be removed, see below
    start_prefix,
    expected_keys,
    device_map=None,
    offload_folder=None,
    offload_index=None,
    state_dict_folder=None,
    state_dict_index=None,
    dtype=None,
644
    is_quantized=False,
Sylvain Gugger's avatar
Sylvain Gugger committed
645
    is_safetensors=False,
646
    keep_in_fp32_modules=None,
647
):
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
    """
    This is somewhat similar to `_load_state_dict_into_model`, but deals with a model that has some or all of its
    params on a `meta` device. It replaces the model params with the data from the `state_dict`, while moving the
    params back to the normal device, but only for `loaded_state_dict_keys`.

    `start_prefix` is used for models which insert their name into model keys, e.g. `bert` in
    `bert.pooler.dense.weight`

    """

    # XXX: remaining features to implement to be fully compatible with _load_state_dict_into_model
    # - deepspeed zero 3 support
    # - need to copy metadata if any - see _load_state_dict_into_model
    # - handling error_msgs - mimicking the error handling in module._load_from_state_dict()
    # - Is there a situation where some keys aren't in `loaded_state_dict_keys` and in which case
    #   they won't get loaded.

665
666
    if is_quantized:
        from .utils.bitsandbytes import set_module_quantized_tensor_to_device
667

668
669
    error_msgs = []

670
671
672
673
674
675
676
677
678
679
680
681
682
    old_keys = []
    new_keys = []
    for key in state_dict.keys():
        new_key = None
        if "gamma" in key:
            new_key = key.replace("gamma", "weight")
        if "beta" in key:
            new_key = key.replace("beta", "bias")
        if new_key:
            old_keys.append(key)
            new_keys.append(new_key)
    for old_key, new_key in zip(old_keys, new_keys):
        state_dict[new_key] = state_dict.pop(old_key)
683

684
685
686
687
688
689
690
691
692
    for param_name, param in state_dict.items():
        # First part of the test is always true as load_state_dict_keys always contains state_dict keys.
        if param_name not in loaded_state_dict_keys or param_name not in expected_keys:
            continue

        if param_name.startswith(start_prefix):
            param_name = param_name[len(start_prefix) :]

        module_name = param_name
693
        set_module_kwargs = {}
694

695
        # We convert floating dtypes to the `dtype` passed. We want to keep the buffers/params
696
697
        # in int/uint/bool and not cast them.
        if dtype is not None and torch.is_floating_point(param):
698
699
700
701
702
703
            if (
                keep_in_fp32_modules is not None
                and any(module_to_keep_in_fp32 in param_name for module_to_keep_in_fp32 in keep_in_fp32_modules)
                and dtype == torch.float16
            ):
                param = param.to(torch.float32)
704
705
706
707
708

                # For backward compatibility with older versions of `accelerate`
                # TODO: @sgugger replace this check with version check at the next `accelerate` release
                if "dtype" in list(inspect.signature(set_module_tensor_to_device).parameters):
                    set_module_kwargs["dtype"] = torch.float32
709
710
            else:
                param = param.to(dtype)
711
712
713
714
715
716
717
718
719
720
721
722

        # For compatibility with PyTorch load_state_dict which converts state dict dtype to existing dtype in model
        if dtype is None:
            old_param = model
            splits = param_name.split(".")
            for split in splits:
                old_param = getattr(old_param, split)
                if old_param is None:
                    break

            if old_param is not None:
                param = param.to(old_param.dtype)
723

724
725
        set_module_kwargs["value"] = param

726
727
728
729
730
731
732
733
734
735
736
        if device_map is None:
            param_device = "cpu"
        else:
            # find next higher level module that is defined in device_map:
            # bert.lm_head.weight -> bert.lm_head -> bert -> ''
            while len(module_name) > 0 and module_name not in device_map:
                module_name = ".".join(module_name.split(".")[:-1])
            if module_name == "" and "" not in device_map:
                # TODO: group all errors and raise at the end.
                raise ValueError(f"{param_name} doesn't have any device set.")
            param_device = device_map[module_name]
737

738
        if param_device == "disk":
Sylvain Gugger's avatar
Sylvain Gugger committed
739
740
            if not is_safetensors:
                offload_index = offload_weight(param, param_name, offload_folder, offload_index)
741
742
        elif param_device == "cpu" and state_dict_index is not None:
            state_dict_index = offload_weight(param, param_name, state_dict_folder, state_dict_index)
743
        elif not is_quantized:
744
745
            # For backward compatibility with older versions of `accelerate`
            set_module_tensor_to_device(model, param_name, param_device, **set_module_kwargs)
746
        else:
747
748
749
750
751
752
            if param.dtype == torch.int8 and param_name.replace("weight", "SCB") in state_dict.keys():
                fp16_statistics = state_dict[param_name.replace("weight", "SCB")]
            else:
                fp16_statistics = None

            if "SCB" not in param_name:
753
                set_module_quantized_tensor_to_device(
754
755
                    model, param_name, param_device, value=param, fp16_statistics=fp16_statistics
                )
756
757

    return error_msgs, offload_index, state_dict_index
758
759


760
761
762
763
764
765
766
767
768
def _add_variant(weights_name: str, variant: Optional[str] = None) -> str:
    if variant is not None:
        splits = weights_name.split(".")
        splits = splits[:-1] + [variant] + splits[-1:]
        weights_name = ".".join(splits)

    return weights_name


769
class ModuleUtilsMixin:
Julien Chaumond's avatar
Julien Chaumond committed
770
    """
771
    A few utilities for `torch.nn.Modules`, to be used as a mixin.
Julien Chaumond's avatar
Julien Chaumond committed
772
773
    """

774
775
776
777
    @staticmethod
    def _hook_rss_memory_pre_forward(module, *args, **kwargs):
        try:
            import psutil
Sylvain Gugger's avatar
Sylvain Gugger committed
778
        except ImportError:
779
780
781
782
783
784
785
786
787
788
789
            raise ImportError("You need to install psutil (pip install psutil) to use memory tracing.")

        process = psutil.Process(os.getpid())
        mem = process.memory_info()
        module.mem_rss_pre_forward = mem.rss
        return None

    @staticmethod
    def _hook_rss_memory_post_forward(module, *args, **kwargs):
        try:
            import psutil
Sylvain Gugger's avatar
Sylvain Gugger committed
790
        except ImportError:
791
792
793
794
795
796
797
798
799
800
            raise ImportError("You need to install psutil (pip install psutil) to use memory tracing.")

        process = psutil.Process(os.getpid())
        mem = process.memory_info()
        module.mem_rss_post_forward = mem.rss
        mem_rss_diff = module.mem_rss_post_forward - module.mem_rss_pre_forward
        module.mem_rss_diff = mem_rss_diff + (module.mem_rss_diff if hasattr(module, "mem_rss_diff") else 0)
        return None

    def add_memory_hooks(self):
Sylvain Gugger's avatar
Sylvain Gugger committed
801
802
803
        """
        Add a memory hook before and after each sub-module forward pass to record increase in memory consumption.

Sylvain Gugger's avatar
Sylvain Gugger committed
804
805
        Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to zero
        with `model.reset_memory_hooks_state()`.
806
807
808
809
810
811
812
        """
        for module in self.modules():
            module.register_forward_pre_hook(self._hook_rss_memory_pre_forward)
            module.register_forward_hook(self._hook_rss_memory_post_forward)
        self.reset_memory_hooks_state()

    def reset_memory_hooks_state(self):
Sylvain Gugger's avatar
Sylvain Gugger committed
813
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
814
        Reset the `mem_rss_diff` attribute of each module (see [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]).
Sylvain Gugger's avatar
Sylvain Gugger committed
815
        """
816
817
818
819
820
        for module in self.modules():
            module.mem_rss_diff = 0
            module.mem_rss_post_forward = 0
            module.mem_rss_pre_forward = 0

821
    @property
Sylvain Gugger's avatar
Sylvain Gugger committed
822
    def device(self) -> torch.device:
823
        """
824
        `torch.device`: The device on which the module is (assuming that all the module parameters are on the same
825
        device).
826
        """
Lysandre Debut's avatar
Lysandre Debut committed
827
        return get_parameter_device(self)
828

829
    @property
830
    def dtype(self) -> torch.dtype:
831
        """
832
        `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
833
        """
Lysandre Debut's avatar
Lysandre Debut committed
834
        return get_parameter_dtype(self)
835
836

    def invert_attention_mask(self, encoder_attention_mask: Tensor) -> Tensor:
Sylvain Gugger's avatar
Sylvain Gugger committed
837
838
839
840
        """
        Invert an attention mask (e.g., switches 0. and 1.).

        Args:
841
            encoder_attention_mask (`torch.Tensor`): An attention mask.
Sylvain Gugger's avatar
Sylvain Gugger committed
842
843

        Returns:
844
            `torch.Tensor`: The inverted attention mask.
Sylvain Gugger's avatar
Sylvain Gugger committed
845
        """
846
847
848
849
850
851
852
853
854
855
        if encoder_attention_mask.dim() == 3:
            encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
        if encoder_attention_mask.dim() == 2:
            encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
        # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
        # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
        # /transformer/transformer_layers.py#L270
        # encoder_extended_attention_mask = (encoder_extended_attention_mask ==
        # encoder_extended_attention_mask.transpose(-1, -2))
        encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
Yih-Dar's avatar
Yih-Dar committed
856
        encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * torch.finfo(self.dtype).min
857

858
859
        return encoder_extended_attention_mask

860
    @staticmethod
861
862
863
864
865
866
867
    def create_extended_attention_mask_for_decoder(input_shape, attention_mask, device=None):
        if device is not None:
            warnings.warn(
                "The `device` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
            )
        else:
            device = attention_mask.device
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
        batch_size, seq_length = input_shape
        seq_ids = torch.arange(seq_length, device=device)
        causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
        # in case past_key_values are used we need to add a prefix ones mask to the causal mask
        # causal and attention masks must have same type with pytorch version < 1.3
        causal_mask = causal_mask.to(attention_mask.dtype)

        if causal_mask.shape[1] < attention_mask.shape[1]:
            prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
            causal_mask = torch.cat(
                [
                    torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
                    causal_mask,
                ],
                axis=-1,
            )

        extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
        return extended_attention_mask

888
    def get_extended_attention_mask(
889
        self, attention_mask: Tensor, input_shape: Tuple[int], device: torch.device = None, dtype: torch.float = None
890
    ) -> Tensor:
Sylvain Gugger's avatar
Sylvain Gugger committed
891
892
        """
        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
893
894

        Arguments:
895
            attention_mask (`torch.Tensor`):
Sylvain Gugger's avatar
Sylvain Gugger committed
896
                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
897
            input_shape (`Tuple[int]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
898
                The shape of the input to the model.
899
900

        Returns:
901
            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
902
        """
Yih-Dar's avatar
Yih-Dar committed
903
904
905
        if dtype is None:
            dtype = self.dtype

906
907
908
909
910
911
        if not (attention_mask.dim() == 2 and self.config.is_decoder):
            # show warning only if it won't be shown in `create_extended_attention_mask_for_decoder`
            if device is not None:
                warnings.warn(
                    "The `device` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
                )
912
913
914
915
916
917
918
919
920
        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        if attention_mask.dim() == 3:
            extended_attention_mask = attention_mask[:, None, :, :]
        elif attention_mask.dim() == 2:
            # Provided a padding mask of dimensions [batch_size, seq_length]
            # - if the model is a decoder, apply a causal mask in addition to the padding mask
            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
            if self.config.is_decoder:
921
                extended_attention_mask = ModuleUtilsMixin.create_extended_attention_mask_for_decoder(
922
923
                    input_shape, attention_mask, device
                )
924
925
926
927
            else:
                extended_attention_mask = attention_mask[:, None, None, :]
        else:
            raise ValueError(
928
                f"Wrong shape for input_ids (shape {input_shape}) or attention_mask (shape {attention_mask.shape})"
929
930
931
932
            )

        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
        # masked positions, this operation will create a tensor which is 0.0 for
933
        # positions we want to attend and the dtype's smallest value for masked positions.
934
935
        # Since we are adding it to the raw scores before the softmax, this is
        # effectively the same as removing these entirely.
Yih-Dar's avatar
Yih-Dar committed
936
937
        extended_attention_mask = extended_attention_mask.to(dtype=dtype)  # fp16 compatibility
        extended_attention_mask = (1.0 - extended_attention_mask) * torch.finfo(dtype).min
938
939
        return extended_attention_mask

Sylvain Gugger's avatar
Sylvain Gugger committed
940
941
942
    def get_head_mask(
        self, head_mask: Optional[Tensor], num_hidden_layers: int, is_attention_chunked: bool = False
    ) -> Tensor:
943
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
944
945
946
        Prepare the head mask if needed.

        Args:
947
            head_mask (`torch.Tensor` with shape `[num_heads]` or `[num_hidden_layers x num_heads]`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
948
                The mask indicating if we should keep the heads or not (1.0 for keep, 0.0 for discard).
949
            num_hidden_layers (`int`):
Sylvain Gugger's avatar
Sylvain Gugger committed
950
                The number of hidden layers in the model.
951
            is_attention_chunked (`bool`, *optional*, defaults to `False`):
Sylvain Gugger's avatar
Sylvain Gugger committed
952
953
                Whether or not the attentions scores are computed by chunks or not.

954
        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
955
956
            `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or list with
            `[None]` for each layer.
957
958
959
        """
        if head_mask is not None:
            head_mask = self._convert_head_mask_to_5d(head_mask, num_hidden_layers)
Patrick von Platen's avatar
Patrick von Platen committed
960
961
            if is_attention_chunked is True:
                head_mask = head_mask.unsqueeze(-1)
962
963
964
965
966
967
968
969
970
971
972
973
974
        else:
            head_mask = [None] * num_hidden_layers

        return head_mask

    def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers):
        """-> [num_hidden_layers x batch x num_heads x seq_length x seq_length]"""
        if head_mask.dim() == 1:
            head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
            head_mask = head_mask.expand(num_hidden_layers, -1, -1, -1, -1)
        elif head_mask.dim() == 2:
            head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
        assert head_mask.dim() == 5, f"head_mask.dim != 5, instead {head_mask.dim()}"
975
        head_mask = head_mask.to(dtype=self.dtype)  # switch to float if need + fp16 compatibility
976
977
        return head_mask

978
979
980
981
982
    def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int:
        """
        Get number of (optionally, trainable or non-embeddings) parameters in the module.

        Args:
983
            only_trainable (`bool`, *optional*, defaults to `False`):
984
985
                Whether or not to return only the number of trainable parameters

986
            exclude_embeddings (`bool`, *optional*, defaults to `False`):
987
988
989
                Whether or not to return only the number of non-embeddings parameters

        Returns:
990
            `int`: The number of parameters.
991
992
        """

993
994
995
996
997
998
999
1000
1001
1002
        if exclude_embeddings:
            embedding_param_names = [
                f"{name}.weight" for name, module_type in self.named_modules() if isinstance(module_type, nn.Embedding)
            ]
            non_embedding_parameters = [
                parameter for name, parameter in self.named_parameters() if name not in embedding_param_names
            ]
            return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable)
        else:
            return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable)
1003
1004
1005
1006
1007
1008

    def estimate_tokens(self, input_dict: Dict[str, Union[torch.Tensor, Any]]) -> int:
        """
        Helper function to estimate the total number of tokens from the model inputs.

        Args:
1009
            inputs (`dict`): The model inputs.
1010
1011

        Returns:
1012
            `int`: The total number of tokens.
1013
        """
1014
1015
        if not hasattr(self, "warnings_issued"):
            self.warnings_issued = {}
1016
1017
        if self.main_input_name in input_dict:
            return input_dict[self.main_input_name].numel()
1018
        elif "estimate_tokens" not in self.warnings_issued:
1019
            logger.warning(
1020
1021
                "Could not estimate the number of tokens of the input, floating-point operations will not be computed"
            )
1022
1023
            self.warnings_issued["estimate_tokens"] = True
        return 0
1024
1025
1026
1027
1028
1029
1030

    def floating_point_ops(
        self, input_dict: Dict[str, Union[torch.Tensor, Any]], exclude_embeddings: bool = True
    ) -> int:
        """
        Get number of (optionally, non-embeddings) floating-point operations for the forward and backward passes of a
        batch with this transformer model. Default approximation neglects the quadratic dependency on the number of
Sylvain Gugger's avatar
Sylvain Gugger committed
1031
1032
        tokens (valid if `12 * d_model << sequence_length`) as laid out in [this
        paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter
Sylvain Gugger's avatar
Sylvain Gugger committed
1033
        re-use e.g. Albert or Universal Transformers, or if doing long-range modeling with very high sequence lengths.
1034
1035

        Args:
1036
            batch_size (`int`):
1037
1038
                The batch size for the forward pass.

1039
            sequence_length (`int`):
1040
1041
                The number of tokens in each line of the batch.

1042
            exclude_embeddings (`bool`, *optional*, defaults to `True`):
1043
1044
1045
                Whether or not to count embedding and softmax operations.

        Returns:
1046
            `int`: The number of floating-point operations.
1047
1048
1049
1050
        """

        return 6 * self.estimate_tokens(input_dict) * self.num_parameters(exclude_embeddings=exclude_embeddings)

Julien Chaumond's avatar
Julien Chaumond committed
1051

1052
class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMixin, PeftAdapterMixin):
1053
1054
    r"""
    Base class for all models.
1055

Sylvain Gugger's avatar
Sylvain Gugger committed
1056
1057
    [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading,
    downloading and saving models as well as a few methods common to all models to:
1058

1059
1060
        - resize the input embeddings,
        - prune heads in the self-attention heads.
1061

1062
    Class attributes (overridden by derived classes):
Sylvain Gugger's avatar
Sylvain Gugger committed
1063

Sylvain Gugger's avatar
Sylvain Gugger committed
1064
1065
1066
1067
        - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class
          for this model architecture.
        - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch model,
          taking as arguments:
1068

Sylvain Gugger's avatar
Sylvain Gugger committed
1069
1070
            - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the TensorFlow checkpoint.
            - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to the model.
1071
            - **path** (`str`) -- A path to the TensorFlow checkpoint.
1072

Sylvain Gugger's avatar
Sylvain Gugger committed
1073
1074
        - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived
          classes of the same architecture adding modules on top of the base model.
1075
        - **is_parallelizable** (`bool`) -- A flag indicating whether this model supports model parallelization.
Sylvain Gugger's avatar
Sylvain Gugger committed
1076
1077
        - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP
          models, `pixel_values` for vision models and `input_values` for speech models).
1078
    """
1079
    config_class = None
1080
    base_model_prefix = ""
1081
    main_input_name = "input_ids"
1082
    _auto_class = None
1083
    _no_split_modules = None
1084
    _skip_keys_device_placement = None
1085
    _keep_in_fp32_modules = None
1086

1087
1088
    # a list of `re` patterns of `state_dict` keys that should be removed from the list of missing
    # keys we find (keys inside the model but not in the checkpoint) and avoid unnecessary warnings.
1089
    _keys_to_ignore_on_load_missing = None
1090
1091
1092
    # a list of `re` patterns of `state_dict` keys that should be removed from the list of
    # unexpected keys we find (keys inside the checkpoint but not the model) and avoid unnecessary
    # warnings.
1093
    _keys_to_ignore_on_load_unexpected = None
1094
1095
    # a list of `state_dict` keys to ignore when saving the model (useful for keys that aren't
    # trained, but which are either deterministic or tied variables)
1096
    _keys_to_ignore_on_save = None
Sylvain Gugger's avatar
Sylvain Gugger committed
1097
1098
    # a list of `state_dict` keys that are potentially tied to another key in the state_dict.
    _tied_weights_keys = None
1099

1100
    is_parallelizable = False
1101
    supports_gradient_checkpointing = False
1102

1103
    @property
1104
    def dummy_inputs(self) -> Dict[str, torch.Tensor]:
1105
        """
1106
        `Dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network.
1107
        """
1108
        return {"input_ids": torch.tensor(DUMMY_INPUTS)}
1109

1110
1111
1112
1113
1114
1115
1116
    @property
    def framework(self) -> str:
        """
        :str: Identifies that this is a PyTorch model.
        """
        return "pt"

1117
    def __init__(self, config: PretrainedConfig, *inputs, **kwargs):
Julien Chaumond's avatar
Julien Chaumond committed
1118
        super().__init__()
1119
1120
        if not isinstance(config, PretrainedConfig):
            raise ValueError(
1121
1122
1123
                f"Parameter config in `{self.__class__.__name__}(config)` should be an instance of class "
                "`PretrainedConfig`. To create a model from a pretrained model use "
                f"`model = {self.__class__.__name__}.from_pretrained(PRETRAINED_MODEL_NAME)`"
1124
            )
1125
        # Save config and origin of the pretrained weights if given in model
1126
        self.config = config
1127
        self.name_or_path = config.name_or_path
1128
        self.warnings_issued = {}
1129
        self.generation_config = GenerationConfig.from_model_config(config) if self.can_generate() else None
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143

    def post_init(self):
        """
        A method executed at the end of each Transformer model initialization, to execute code that needs the model's
        modules properly initialized (such as weight initialization).
        """
        self.init_weights()
        self._backward_compatibility_gradient_checkpointing()

    def _backward_compatibility_gradient_checkpointing(self):
        if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False):
            self.gradient_checkpointing_enable()
            # Remove the attribute now that is has been consumed, so it's no saved in the config.
            delattr(self.config, "gradient_checkpointing")
1144

1145
1146
1147
1148
1149
1150
    @classmethod
    def _from_config(cls, config, **kwargs):
        """
        All context managers that the model should be initialized under go here.

        Args:
1151
1152
            torch_dtype (`torch.dtype`, *optional*):
                Override the default `torch.dtype` and load the model under this dtype.
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
        """
        torch_dtype = kwargs.pop("torch_dtype", None)

        # override default dtype if needed
        dtype_orig = None
        if torch_dtype is not None:
            dtype_orig = cls._set_default_torch_dtype(torch_dtype)

        if is_deepspeed_zero3_enabled():
            import deepspeed

            logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
            # this immediately partitions the model across all gpus, to avoid the overhead in time
            # and memory copying it on CPU or each GPU first
1167
            with deepspeed.zero.Init(config_dict_or_path=deepspeed_config()):
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
                model = cls(config, **kwargs)
        else:
            model = cls(config, **kwargs)

        # restore default dtype if it was modified
        if dtype_orig is not None:
            torch.set_default_dtype(dtype_orig)

        return model

    @classmethod
    def _set_default_torch_dtype(cls, dtype: torch.dtype) -> torch.dtype:
        """
        Change the default dtype and return the previous one. This is needed when wanting to instantiate the model
        under specific dtype.

        Args:
1185
            dtype (`torch.dtype`):
1186
1187
1188
                a floating dtype to set to.

        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
1189
1190
            `torch.dtype`: the original `dtype` that can be used to restore `torch.set_default_dtype(dtype)` if it was
            modified. If it wasn't, returns `None`.
1191

1192
1193
        Note `set_default_dtype` currently only works with floating-point types and asserts if for example,
        `torch.int64` is passed. So if a non-float `dtype` is passed this functions will throw an exception.
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
        """
        if not dtype.is_floating_point:
            raise ValueError(
                f"Can't instantiate {cls.__name__} model under dtype={dtype} since it is not a floating point dtype"
            )

        logger.info(f"Instantiating {cls.__name__} model under default dtype {dtype}.")
        dtype_orig = torch.get_default_dtype()
        torch.set_default_dtype(dtype)
        return dtype_orig

1205
    @property
1206
1207
    def base_model(self) -> nn.Module:
        """
1208
        `torch.nn.Module`: The main body of the model.
1209
        """
1210
        return getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
1211

1212
1213
    @classmethod
    def can_generate(cls) -> bool:
1214
1215
1216
1217
1218
1219
1220
        """
        Returns whether this model can generate sequences with `.generate()`.

        Returns:
            `bool`: Whether this model can generate sequences with `.generate()`.
        """
        # Detects whether `prepare_inputs_for_generation` has been overwritten, which is a requirement for generation
1221
        if "GenerationMixin" in str(cls.prepare_inputs_for_generation):
1222
1223
1224
            return False
        return True

1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
    def enable_input_require_grads(self):
        """
        Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping
        the model weights fixed.
        """

        def make_inputs_require_grads(module, input, output):
            output.requires_grad_(True)

        self._require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)

    def disable_input_require_grads(self):
        """
        Removes the `_require_grads_hook`.
        """
        self._require_grads_hook.remove()

1242
    def get_input_embeddings(self) -> nn.Module:
1243
1244
1245
1246
        """
        Returns the model's input embeddings.

        Returns:
1247
            `nn.Module`: A torch module mapping vocabulary to hidden states.
thomwolf's avatar
thomwolf committed
1248
        """
1249
        base_model = getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
1250
1251
1252
1253
        if base_model is not self:
            return base_model.get_input_embeddings()
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
1254

1255
    def set_input_embeddings(self, value: nn.Module):
1256
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
1257
        Set model's input embeddings.
1258
1259

        Args:
1260
            value (`nn.Module`): A module mapping vocabulary to hidden states.
thomwolf's avatar
thomwolf committed
1261
1262
1263
1264
1265
1266
        """
        base_model = getattr(self, self.base_model_prefix, self)
        if base_model is not self:
            base_model.set_input_embeddings(value)
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
1267

1268
    def get_output_embeddings(self) -> nn.Module:
1269
1270
1271
1272
        """
        Returns the model's output embeddings.

        Returns:
1273
            `nn.Module`: A torch module mapping hidden states to vocabulary.
thomwolf's avatar
thomwolf committed
1274
        """
1275
        return None  # Overwrite for models with output embeddings
thomwolf's avatar
thomwolf committed
1276

1277
1278
1279
1280
    def _init_weights(self, module):
        """
        Initialize the weights. This method should be overridden by derived class.
        """
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
        pass

    def _initialize_weights(self, module):
        """
        Initialize the weights if they are not already initialized.
        """
        if getattr(module, "_is_hf_initialized", False):
            return
        self._init_weights(module)
        module._is_hf_initialized = True
1291

1292
    def tie_weights(self):
1293
1294
        """
        Tie the weights between the input embeddings and the output embeddings.
1295

Sylvain Gugger's avatar
Sylvain Gugger committed
1296
1297
        If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the
        weights instead.
thomwolf's avatar
thomwolf committed
1298
        """
1299
1300
1301
1302
        if getattr(self.config, "tie_word_embeddings", True):
            output_embeddings = self.get_output_embeddings()
            if output_embeddings is not None:
                self._tie_or_clone_weights(output_embeddings, self.get_input_embeddings())
thomwolf's avatar
thomwolf committed
1303

1304
        if getattr(self.config, "is_encoder_decoder", False) and getattr(self.config, "tie_encoder_decoder", False):
Weizhen's avatar
Weizhen committed
1305
1306
            if hasattr(self, self.base_model_prefix):
                self = getattr(self, self.base_model_prefix)
1307
1308
            self._tie_encoder_decoder_weights(self.encoder, self.decoder, self.base_model_prefix)

Sylvain Gugger's avatar
Sylvain Gugger committed
1309
1310
1311
1312
        for module in self.modules():
            if hasattr(module, "_tie_weights"):
                module._tie_weights()

1313
1314
1315
    @staticmethod
    def _tie_encoder_decoder_weights(encoder: nn.Module, decoder: nn.Module, base_model_prefix: str):
        uninitialized_encoder_weights: List[str] = []
Weizhen's avatar
Weizhen committed
1316
1317
        if decoder.__class__ != encoder.__class__:
            logger.info(
Sylvain Gugger's avatar
Sylvain Gugger committed
1318
1319
                f"{decoder.__class__} and {encoder.__class__} are not equal. In this case make sure that all encoder"
                " weights are correctly initialized."
Weizhen's avatar
Weizhen committed
1320
            )
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330

        def tie_encoder_to_decoder_recursively(
            decoder_pointer: nn.Module,
            encoder_pointer: nn.Module,
            module_name: str,
            uninitialized_encoder_weights: List[str],
            depth=0,
        ):
            assert isinstance(decoder_pointer, nn.Module) and isinstance(
                encoder_pointer, nn.Module
1331
            ), f"{decoder_pointer} and {encoder_pointer} have to be of type nn.Module"
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
            if hasattr(decoder_pointer, "weight"):
                assert hasattr(encoder_pointer, "weight")
                encoder_pointer.weight = decoder_pointer.weight
                if hasattr(decoder_pointer, "bias"):
                    assert hasattr(encoder_pointer, "bias")
                    encoder_pointer.bias = decoder_pointer.bias
                return

            encoder_modules = encoder_pointer._modules
            decoder_modules = decoder_pointer._modules
            if len(decoder_modules) > 0:
                assert (
                    len(encoder_modules) > 0
                ), f"Encoder module {encoder_pointer} does not match decoder module {decoder_pointer}"

1347
                all_encoder_weights = {module_name + "/" + sub_name for sub_name in encoder_modules.keys()}
1348
1349
1350
1351
1352
                encoder_layer_pos = 0
                for name, module in decoder_modules.items():
                    if name.isdigit():
                        encoder_name = str(int(name) + encoder_layer_pos)
                        decoder_name = name
Weizhen's avatar
Weizhen committed
1353
1354
1355
                        if not isinstance(decoder_modules[decoder_name], type(encoder_modules[encoder_name])) and len(
                            encoder_modules
                        ) != len(decoder_modules):
1356
1357
                            # this can happen if the name corresponds to the position in a list module list of layers
                            # in this case the decoder has added a cross-attention that the encoder does not have
1358
                            # thus skip this step and subtract one layer pos from encoder
1359
1360
1361
1362
1363
1364
                            encoder_layer_pos -= 1
                            continue
                    elif name not in encoder_modules:
                        continue
                    elif depth > 500:
                        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1365
1366
                            "Max depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is"
                            " a circular dependency between two or more `nn.Modules` of your model."
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
                        )
                    else:
                        decoder_name = encoder_name = name
                    tie_encoder_to_decoder_recursively(
                        decoder_modules[decoder_name],
                        encoder_modules[encoder_name],
                        module_name + "/" + name,
                        uninitialized_encoder_weights,
                        depth=depth + 1,
                    )
                    all_encoder_weights.remove(module_name + "/" + encoder_name)

                uninitialized_encoder_weights += list(all_encoder_weights)

        # tie weights recursively
        tie_encoder_to_decoder_recursively(decoder, encoder, base_model_prefix, uninitialized_encoder_weights)
        if len(uninitialized_encoder_weights) > 0:
            logger.warning(
                f"The following encoder weights were not tied to the decoder {uninitialized_encoder_weights}"
            )

1388
    def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
Lysandre's avatar
Lysandre committed
1389
        """Tie or clone module weights depending of whether we are using TorchScript or not"""
thomwolf's avatar
thomwolf committed
1390
        if self.config.torchscript:
1391
            output_embeddings.weight = nn.Parameter(input_embeddings.weight.clone())
thomwolf's avatar
thomwolf committed
1392
        else:
1393
            output_embeddings.weight = input_embeddings.weight
thomwolf's avatar
thomwolf committed
1394

Sam Shleifer's avatar
Sam Shleifer committed
1395
        if getattr(output_embeddings, "bias", None) is not None:
1396
            output_embeddings.bias.data = nn.functional.pad(
1397
                output_embeddings.bias.data,
Lysandre's avatar
Lysandre committed
1398
1399
1400
1401
                (
                    0,
                    output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0],
                ),
1402
1403
                "constant",
                0,
1404
            )
1405
        if hasattr(output_embeddings, "out_features") and hasattr(input_embeddings, "num_embeddings"):
1406
            output_embeddings.out_features = input_embeddings.num_embeddings
1407

1408
1409
1410
    def resize_token_embeddings(
        self, new_num_tokens: Optional[int] = None, pad_to_multiple_of: Optional[int] = None
    ) -> nn.Embedding:
1411
        """
1412
        Resizes input token embeddings matrix of the model if `new_num_tokens != config.vocab_size`.
1413

1414
        Takes care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
thomwolf's avatar
thomwolf committed
1415

1416
        Arguments:
1417
            new_num_tokens (`int`, *optional*):
1418
                The number of new tokens in the embedding matrix. Increasing the size will add newly initialized
Sylvain Gugger's avatar
Sylvain Gugger committed
1419
1420
                vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just
                returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything.
1421
1422
1423
1424
1425
1426
1427
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the embedding matrix to a multiple of the provided value.

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
                details about this, or help on choosing the correct value for resizing, refer to this guide:
                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc
1428
1429

        Return:
1430
            `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model.
thomwolf's avatar
thomwolf committed
1431
        """
1432
        model_embeds = self._resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
thomwolf's avatar
thomwolf committed
1433
1434
        if new_num_tokens is None:
            return model_embeds
thomwolf's avatar
thomwolf committed
1435
1436
1437

        # Update base model and current model config
        self.config.vocab_size = new_num_tokens
1438
        self.vocab_size = new_num_tokens
thomwolf's avatar
thomwolf committed
1439
1440

        # Tie weights again if needed
1441
        self.tie_weights()
thomwolf's avatar
thomwolf committed
1442

thomwolf's avatar
thomwolf committed
1443
1444
        return model_embeds

1445
    def _resize_token_embeddings(self, new_num_tokens, pad_to_multiple_of=None):
thomwolf's avatar
thomwolf committed
1446
        old_embeddings = self.get_input_embeddings()
1447
        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens, pad_to_multiple_of)
1448
1449
1450
        if hasattr(old_embeddings, "_hf_hook"):
            hook = old_embeddings._hf_hook
            add_hook_to_module(new_embeddings, hook)
thomwolf's avatar
thomwolf committed
1451
        self.set_input_embeddings(new_embeddings)
1452
1453
1454
1455

        # if word embeddings are not tied, make sure that lm head is resized as well
        if self.get_output_embeddings() is not None and not self.config.tie_word_embeddings:
            old_lm_head = self.get_output_embeddings()
1456
            new_lm_head = self._get_resized_lm_head(old_lm_head, new_embeddings.weight.shape[0])
1457
1458
1459
            if hasattr(old_lm_head, "_hf_hook"):
                hook = old_lm_head._hf_hook
                add_hook_to_module(new_lm_head, hook)
1460
1461
            self.set_output_embeddings(new_lm_head)

thomwolf's avatar
thomwolf committed
1462
        return self.get_input_embeddings()
1463

1464
    def _get_resized_embeddings(
1465
1466
1467
1468
        self,
        old_embeddings: nn.Embedding,
        new_num_tokens: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
1469
    ) -> nn.Embedding:
1470
1471
1472
        """
        Build a resized Embedding Module from a provided token Embedding Module. Increasing the size will add newly
        initialized vectors at the end. Reducing the size will remove vectors from the end
1473
1474

        Args:
1475
            old_embeddings (`torch.nn.Embedding`):
1476
                Old embeddings to be resized.
1477
            new_num_tokens (`int`, *optional*):
1478
                New number of tokens in the embedding matrix.
1479
1480

                Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
1481
                vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
1482
                `torch.nn.Embedding` module of the model without doing anything.
1483
1484
1485
1486
1487
1488
1489
1490
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the embedding matrix to a multiple of the provided value.

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
                details about this, or help on choosing the correct value for resizing, refer to this guide:
                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc

1491
1492

        Return:
1493
1494
            `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if
            `new_num_tokens` is `None`
1495
        """
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506

        if pad_to_multiple_of is not None:
            if not isinstance(pad_to_multiple_of, int):
                raise ValueError(
                    f"Asking to pad the embedding matrix to a multiple of `{pad_to_multiple_of}`, which is not and integer. Please make sure to pass an integer"
                )
            if new_num_tokens is None:
                new_num_tokens = old_embeddings.weight.shape[0]
            new_num_tokens = ((new_num_tokens // pad_to_multiple_of) + 1) * pad_to_multiple_of
        else:
            logger.warning(
1507
                "You are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding"
1508
                f" dimension will be {new_num_tokens}. This might induce some performance reduction as *Tensor Cores* will not be available."
1509
                " For more details about this, or help on choosing the correct value for resizing, refer to this guide:"
1510
1511
1512
                " https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc"
            )

1513
1514
1515
        if new_num_tokens is None:
            return old_embeddings

1516
1517
1518
1519
1520
1521
1522
1523
        if is_deepspeed_zero3_enabled():
            import deepspeed

            with deepspeed.zero.GatheredParameters(old_embeddings.weight, modifier_rank=None):
                old_num_tokens, old_embedding_dim = old_embeddings.weight.size()
        else:
            old_num_tokens, old_embedding_dim = old_embeddings.weight.size()

1524
1525
1526
        if old_num_tokens == new_num_tokens:
            return old_embeddings

1527
1528
        if not isinstance(old_embeddings, nn.Embedding):
            raise TypeError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1529
1530
1531
                f"Old embeddings are of type {type(old_embeddings)}, which is not an instance of {nn.Embedding}. You"
                " should either use a different resize function or make sure that `old_embeddings` are an instance of"
                f" {nn.Embedding}."
1532
1533
            )

1534
1535
1536
1537
1538
        # numbers of tokens to copy
        n = min(old_num_tokens, new_num_tokens)
        if is_deepspeed_zero3_enabled():
            import deepspeed

1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
            with deepspeed.zero.Init(config_dict_or_path=deepspeed_config()):
                # Build new embeddings
                new_embeddings = nn.Embedding(
                    new_num_tokens,
                    old_embedding_dim,
                    device=old_embeddings.weight.device,
                    dtype=old_embeddings.weight.dtype,
                )

            params = [old_embeddings.weight, new_embeddings.weight]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
                # initialize all new embeddings (in particular added tokens)
                self._init_weights(new_embeddings)

                # Copy token embeddings from the previous weights
                new_embeddings.weight.data[:n, :] = old_embeddings.weight.data[:n, :]
1555
        else:
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
            # Build new embeddings
            new_embeddings = nn.Embedding(
                new_num_tokens,
                old_embedding_dim,
                device=old_embeddings.weight.device,
                dtype=old_embeddings.weight.dtype,
            )

            # initialize all new embeddings (in particular added tokens)
            self._init_weights(new_embeddings)

            # Copy token embeddings from the previous weights
1568
            new_embeddings.weight.data[:n, :] = old_embeddings.weight.data[:n, :]
1569
1570
1571

        return new_embeddings

1572
    def _get_resized_lm_head(
1573
1574
        self, old_lm_head: nn.Linear, new_num_tokens: Optional[int] = None, transposed: Optional[bool] = False
    ) -> nn.Linear:
1575
1576
1577
1578
1579
        """
        Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized
        vectors at the end. Reducing the size will remove vectors from the end

        Args:
1580
            old_lm_head (`torch.nn.Linear`):
1581
                Old lm head liner layer to be resized.
1582
            new_num_tokens (`int`, *optional*):
1583
1584
1585
                New number of tokens in the linear matrix.

                Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
1586
                vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
1587
1588
1589
                `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults
                to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim,
                vocab_size` else `vocab_size, lm_head_dim`.
1590
1591

        Return:
Sylvain Gugger's avatar
Sylvain Gugger committed
1592
1593
            `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is
            `None`
1594
1595
1596
1597
        """
        if new_num_tokens is None:
            return old_lm_head

1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
        if is_deepspeed_zero3_enabled():
            import deepspeed

            with deepspeed.zero.GatheredParameters(old_lm_head.weight, modifier_rank=None):
                old_num_tokens, old_lm_head_dim = (
                    old_lm_head.weight.size() if not transposed else old_lm_head.weight.t().size()
                )
        else:
            old_num_tokens, old_lm_head_dim = (
                old_lm_head.weight.size() if not transposed else old_lm_head.weight.t().size()
            )
1609
1610
1611
1612
1613
1614

        if old_num_tokens == new_num_tokens:
            return old_lm_head

        if not isinstance(old_lm_head, nn.Linear):
            raise TypeError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1615
1616
1617
                f"Old language model head is of type {type(old_lm_head)}, which is not an instance of {nn.Linear}. You"
                " should either use a different resize function or make sure that `old_lm_head` are an instance of"
                f" {nn.Linear}."
1618
1619
1620
1621
1622
1623
1624
1625
            )

        # Build new lm head
        new_lm_head_shape = (old_lm_head_dim, new_num_tokens) if not transposed else (new_num_tokens, old_lm_head_dim)
        has_new_lm_head_bias = old_lm_head.bias is not None

        num_tokens_to_copy = min(old_num_tokens, new_num_tokens)

1626
1627
1628
1629
        # XXX: put the long block of code in a wrapper
        if is_deepspeed_zero3_enabled():
            import deepspeed

1630
1631
1632
1633
1634
1635
1636
            with deepspeed.zero.Init(config_dict_or_path=deepspeed_config()):
                new_lm_head = nn.Linear(
                    *new_lm_head_shape,
                    bias=has_new_lm_head_bias,
                    device=old_lm_head.weight.device,
                    dtype=old_lm_head.weight.dtype,
                )
1637
1638
            params = [old_lm_head.weight, old_lm_head.bias, new_lm_head.weight, new_lm_head.bias]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
1639
1640
1641
1642
1643
1644
                self._init_weights(new_lm_head)
                # Copy old lm head weights to new lm head
                if not transposed:
                    new_lm_head.weight.data[:num_tokens_to_copy, :] = old_lm_head.weight.data[:num_tokens_to_copy, :]
                else:
                    new_lm_head.weight.data[:, :num_tokens_to_copy] = old_lm_head.weight.data[:, :num_tokens_to_copy]
1645

1646
1647
1648
                # Copy bias weights to new lm head
                if has_new_lm_head_bias:
                    new_lm_head.bias.data[:num_tokens_to_copy] = old_lm_head.bias.data[:num_tokens_to_copy]
1649
        else:
1650
1651
1652
1653
1654
1655
1656
            new_lm_head = nn.Linear(
                *new_lm_head_shape,
                bias=has_new_lm_head_bias,
                device=old_lm_head.weight.device,
                dtype=old_lm_head.weight.dtype,
            )
            self._init_weights(new_lm_head)
1657
1658
1659
1660
1661
            # Copy old lm head weights to new lm head
            if not transposed:
                new_lm_head.weight.data[:num_tokens_to_copy, :] = old_lm_head.weight.data[:num_tokens_to_copy, :]
            else:
                new_lm_head.weight.data[:, :num_tokens_to_copy] = old_lm_head.weight.data[:, :num_tokens_to_copy]
1662

1663
1664
1665
            # Copy bias weights to new lm head
            if has_new_lm_head_bias:
                new_lm_head.bias.data[:num_tokens_to_copy] = old_lm_head.bias.data[:num_tokens_to_copy]
1666
1667
1668

        return new_lm_head

1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
    def resize_position_embeddings(self, new_num_position_embeddings: int):
        raise NotImplementedError(
            f"`resize_position_embeddings` is not implemented for {self.__class__}`. To implement it, you should "
            f"overwrite this method in the class {self.__class__} in `modeling_{self.__class__.__module__}.py`"
        )

    def get_position_embeddings(self) -> Union[nn.Embedding, Tuple[nn.Embedding]]:
        raise NotImplementedError(
            f"`get_position_embeddings` is not implemented for {self.__class__}`. To implement it, you should "
            f"overwrite this method in the class {self.__class__} in `modeling_{self.__class__.__module__}.py`"
        )

1681
    def init_weights(self):
1682
        """
1683
1684
        If needed prunes and maybe initializes weights. If using a custom `PreTrainedModel`, you need to implement any
        initialization logic in `_init_weights`.
1685
        """
1686
1687
1688
1689
        # Prune heads if needed
        if self.config.pruned_heads:
            self.prune_heads(self.config.pruned_heads)

1690
1691
        if _init_weights:
            # Initialize weights
1692
            self.apply(self._initialize_weights)
1693
1694
1695
1696

            # Tie weights should be skipped when not initializing all weights
            # since from_pretrained(...) calls tie weights anyways
            self.tie_weights()
1697

1698
1699
1700
    def prune_heads(self, heads_to_prune: Dict[int, List[int]]):
        """
        Prunes heads of the base model.
1701

1702
        Arguments:
1703
            heads_to_prune (`Dict[int, List[int]]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1704
1705
1706
                Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads
                to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on
                layer 1 and heads 2 and 3 on layer 2.
thomwolf's avatar
thomwolf committed
1707
        """
1708
        # save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads
1709
        for layer, heads in heads_to_prune.items():
1710
1711
1712
            union_heads = set(self.config.pruned_heads.get(layer, [])) | set(heads)
            self.config.pruned_heads[layer] = list(union_heads)  # Unfortunately we have to store it as list for JSON

1713
        self.base_model._prune_heads(heads_to_prune)
thomwolf's avatar
thomwolf committed
1714

1715
    def gradient_checkpointing_enable(self):
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
        """
        Activates gradient checkpointing for the current model.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        if not self.supports_gradient_checkpointing:
            raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
        self.apply(partial(self._set_gradient_checkpointing, value=True))

1726
    def gradient_checkpointing_disable(self):
1727
1728
1729
1730
1731
1732
1733
1734
1735
        """
        Deactivates gradient checkpointing for the current model.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        if self.supports_gradient_checkpointing:
            self.apply(partial(self._set_gradient_checkpointing, value=False))

1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
    @property
    def is_gradient_checkpointing(self) -> bool:
        """
        Whether gradient checkpointing is activated for this model or not.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())

1746
1747
1748
    def save_pretrained(
        self,
        save_directory: Union[str, os.PathLike],
1749
        is_main_process: bool = True,
1750
1751
        state_dict: Optional[dict] = None,
        save_function: Callable = torch.save,
Sylvain Gugger's avatar
Sylvain Gugger committed
1752
        push_to_hub: bool = False,
Sylvain Gugger's avatar
Sylvain Gugger committed
1753
        max_shard_size: Union[int, str] = "10GB",
1754
        safe_serialization: bool = False,
1755
        variant: Optional[str] = None,
1756
        token: Optional[Union[str, bool]] = None,
1757
        save_peft_format: bool = True,
Sylvain Gugger's avatar
Sylvain Gugger committed
1758
        **kwargs,
1759
    ):
1760
1761
        """
        Save a model and its configuration file to a directory, so that it can be re-loaded using the
1762
        [`~PreTrainedModel.from_pretrained`] class method.
1763

1764
        Arguments:
1765
            save_directory (`str` or `os.PathLike`):
1766
                Directory to which to save. Will be created if it doesn't exist.
1767
1768
1769
1770
            is_main_process (`bool`, *optional*, defaults to `True`):
                Whether the process calling this is the main process or not. Useful when in distributed training like
                TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on
                the main process to avoid race conditions.
1771
            state_dict (nested dictionary of `torch.Tensor`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1772
1773
1774
                The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only
                save parts of the model or if special precautions need to be taken when recovering the state dictionary
                of a model (like when using model parallelism).
1775
            save_function (`Callable`):
1776
                The function to use to save the state dictionary. Useful on distributed training like TPUs when one
1777
1778
                need to replace `torch.save` by another method.
            push_to_hub (`bool`, *optional*, defaults to `False`):
1779
1780
1781
                Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
                repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
                namespace).
Sylvain Gugger's avatar
Sylvain Gugger committed
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
            max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`):
                The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
                lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).

                <Tip warning={true}>

                If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
                which will be bigger than `max_shard_size`.

                </Tip>

1793
1794
            safe_serialization (`bool`, *optional*, defaults to `False`):
                Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
1795
1796
            variant (`str`, *optional*):
                If specified, weights are saved in the format pytorch_model.<variant>.bin.
1797
1798
1799
            token (`str` or `bool`, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
1800
1801
1802
1803
            save_peft_format (`bool`, *optional*, defaults to `True`):
                For backward compatibility with PEFT library, in case adapter weights are attached to the model, all
                keys of the state dict of adapters needs to be pre-pended with `base_model.model`. Advanced users can
                disable this behaviours by setting `save_peft_format` to `False`.
1804
            kwargs (`Dict[str, Any]`, *optional*):
1805
                Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
1806
        """
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
        use_auth_token = kwargs.pop("use_auth_token", None)

        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        if token is not None:
            kwargs["token"] = token

1822
        # Checks if the model has been loaded in 8-bit
1823
        if getattr(self, "is_loaded_in_8bit", False) and getattr(self, "is_8bit_serializable", False):
1824
1825
            warnings.warn(
                "You are calling `save_pretrained` to a 8-bit converted model you may likely encounter unexepected"
1826
                " behaviors. If you want to save 8-bit models, make sure to have `bitsandbytes>0.37.2` installed.",
1827
1828
1829
                UserWarning,
            )

1830
1831
1832
1833
1834
        if getattr(self, "is_loaded_in_4bit", False):
            raise NotImplementedError(
                "You are calling `save_pretrained` on a 4-bit converted model. This is currently not supported"
            )

1835
1836
1837
1838
1839
        if "save_config" in kwargs:
            warnings.warn(
                "`save_config` is deprecated and will be removed in v5 of Transformers. Use `is_main_process` instead."
            )
            is_main_process = kwargs.pop("save_config")
1840
1841
        if safe_serialization and not is_safetensors_available():
            raise ImportError("`safe_serialization` requires the `safetensors library: `pip install safetensors`.")
1842

1843
        if os.path.isfile(save_directory):
1844
            logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
1845
            return
1846

1847
1848
        os.makedirs(save_directory, exist_ok=True)

1849
1850
        if push_to_hub:
            commit_message = kwargs.pop("commit_message", None)
1851
            repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
1852
            repo_id = self._create_repo(repo_id, **kwargs)
1853
            files_timestamps = self._get_files_timestamps(save_directory)
1854

Julien Chaumond's avatar
Julien Chaumond committed
1855
        # Only save the model itself if we are using distributed training
1856
        model_to_save = unwrap_model(self)
1857

1858
1859
1860
1861
1862
        # save the string version of dtype to the config, e.g. convert torch.float32 => "float32"
        # we currently don't use this setting automatically, but may start to use with v5
        dtype = get_parameter_dtype(model_to_save)
        model_to_save.config.torch_dtype = str(dtype).split(".")[1]

Julien Chaumond's avatar
Julien Chaumond committed
1863
1864
1865
        # Attach architecture to the config
        model_to_save.config.architectures = [model_to_save.__class__.__name__]

1866
1867
1868
1869
1870
        # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be
        # loaded from the Hub.
        if self._auto_class is not None:
            custom_object_save(self, save_directory, config=self.config)

1871
1872
        _hf_peft_config_loaded = getattr(model_to_save, "_hf_peft_config_loaded", False)

1873
        # Save the config
1874
        if is_main_process:
1875
1876
            if not _hf_peft_config_loaded:
                model_to_save.config.save_pretrained(save_directory)
1877
1878
            if self.can_generate():
                model_to_save.generation_config.save_pretrained(save_directory)
1879

1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
            if _hf_peft_config_loaded:
                logger.info(
                    "Detected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved."
                )
                state_dict = model_to_save.get_adapter_state_dict()

                if save_peft_format:
                    logger.info(
                        "To match the expected format of the PEFT library, all keys of the state dict of adapters will be pre-pended with `base_model.model`."
                    )
                    peft_state_dict = {}
                    for key, value in state_dict.items():
                        peft_state_dict[f"base_model.model.{key}"] = value
                    state_dict = peft_state_dict

                current_peft_config = self.peft_config[self.active_adapter()]
                current_peft_config.save_pretrained(save_directory)

1898
1899
1900
        # Save the model
        if state_dict is None:
            state_dict = model_to_save.state_dict()
1901

1902
1903
1904
1905
1906
        # Translate state_dict from smp to hf if saving with smp >= 1.10
        if IS_SAGEMAKER_MP_POST_1_10:
            for smp_to_hf, _ in smp.state.module_manager.translate_functions:
                state_dict = smp_to_hf(state_dict)

1907
        # Handle the case where some state_dict keys shouldn't be saved
1908
        if self._keys_to_ignore_on_save is not None:
1909
            for ignore_key in self._keys_to_ignore_on_save:
1910
1911
                if ignore_key in state_dict.keys():
                    del state_dict[ignore_key]
1912
1913
1914
1915
1916
        if safe_serialization:
            # Safetensors does not allow tensor aliasing.
            # We're going to remove aliases before saving
            ptrs = collections.defaultdict(list)
            for name, tensor in state_dict.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
1917
                ptrs[id_tensor_storage(tensor)].append(name)
1918
1919
1920
1921
1922
1923
1924

            # These are all the pointers of shared tensors.
            shared_ptrs = {ptr: names for ptr, names in ptrs.items() if len(names) > 1}
            warn_names = set()
            for names in shared_ptrs.values():
                # Removing the keys which are declared as known duplicates on
                # load. This allows to make sure the name which is kept is consistent.
Sylvain Gugger's avatar
Sylvain Gugger committed
1925
                if self._tied_weights_keys is not None:
1926
1927
                    found = 0
                    for name in sorted(names):
Sylvain Gugger's avatar
Sylvain Gugger committed
1928
                        matches_pattern = any(re.search(pat, name) for pat in self._tied_weights_keys)
1929
                        if matches_pattern and name in state_dict:
1930
1931
1932
                            found += 1
                            if found < len(names):
                                del state_dict[name]
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949

                # When not all duplicates have been cleaned, still remove those keys, but put a clear warning.
                # If the link between tensors was done at runtime then `from_pretrained` will not get
                # the key back leading to random tensor. A proper warning will be shown
                # during reload (if applicable), but since the file is not necessarily compatible with
                # the config, better show a proper warning.
                found = 0
                for name in names:
                    if name in state_dict:
                        found += 1
                        if found > 1:
                            del state_dict[name]
                            warn_names.add(name)
            if len(warn_names) > 0:
                logger.warning_once(
                    f"Removed shared tensor {warn_names} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading",
                )
1950

Sylvain Gugger's avatar
Sylvain Gugger committed
1951
        # Shard the model if it is too big.
1952
1953
1954
1955
1956
        if not _hf_peft_config_loaded:
            weights_name = SAFE_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME
            weights_name = _add_variant(weights_name, variant)
        else:
            weights_name = ADAPTER_SAFE_WEIGHTS_NAME if safe_serialization else ADAPTER_WEIGHTS_NAME
1957

1958
        shards, index = shard_checkpoint(state_dict, max_shard_size=max_shard_size, weights_name=weights_name)
Sylvain Gugger's avatar
Sylvain Gugger committed
1959
1960
1961
1962

        # Clean the folder from a previous save
        for filename in os.listdir(save_directory):
            full_filename = os.path.join(save_directory, filename)
1963
1964
            # If we have a shard file that is not going to be replaced, we delete it, but only from the main process
            # in distributed settings to avoid race conditions.
1965
            weights_no_suffix = weights_name.replace(".bin", "").replace(".safetensors", "")
1966
1967
1968

            # make sure that file to be deleted matches format of sharded file, e.g. pytorch_model-00001-of-00005
            filename_no_suffix = filename.replace(".bin", "").replace(".safetensors", "")
1969
            reg = re.compile(r"(.*?)-\d{5}-of-\d{5}")
1970

1971
            if (
1972
                filename.startswith(weights_no_suffix)
1973
1974
1975
                and os.path.isfile(full_filename)
                and filename not in shards.keys()
                and is_main_process
1976
                and reg.fullmatch(filename_no_suffix) is not None
1977
            ):
Sylvain Gugger's avatar
Sylvain Gugger committed
1978
                os.remove(full_filename)
1979

Sylvain Gugger's avatar
Sylvain Gugger committed
1980
1981
        # Save the model
        for shard_file, shard in shards.items():
1982
1983
1984
1985
1986
1987
            if safe_serialization:
                # At some point we will need to deal better with save_function (used for TPU and other distributed
                # joyfulness), but for now this enough.
                safe_save_file(shard, os.path.join(save_directory, shard_file), metadata={"format": "pt"})
            else:
                save_function(shard, os.path.join(save_directory, shard_file))
Sylvain Gugger's avatar
Sylvain Gugger committed
1988
1989

        if index is None:
1990
1991
            path_to_weights = os.path.join(save_directory, _add_variant(WEIGHTS_NAME, variant))
            logger.info(f"Model weights saved in {path_to_weights}")
Sylvain Gugger's avatar
Sylvain Gugger committed
1992
        else:
1993
            save_index_file = SAFE_WEIGHTS_INDEX_NAME if safe_serialization else WEIGHTS_INDEX_NAME
1994
            save_index_file = os.path.join(save_directory, _add_variant(save_index_file, variant))
Sylvain Gugger's avatar
Sylvain Gugger committed
1995
1996
1997
1998
1999
2000
2001
2002
2003
            # Save the index as well
            with open(save_index_file, "w", encoding="utf-8") as f:
                content = json.dumps(index, indent=2, sort_keys=True) + "\n"
                f.write(content)
            logger.info(
                f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be "
                f"split in {len(shards)} checkpoint shards. You can find where each parameters has been saved in the "
                f"index located at {save_index_file}."
            )
2004

Sylvain Gugger's avatar
Sylvain Gugger committed
2005
        if push_to_hub:
2006
            self._upload_modified_files(
2007
2008
2009
2010
                save_directory,
                repo_id,
                files_timestamps,
                commit_message=commit_message,
2011
                token=token,
2012
            )
Sylvain Gugger's avatar
Sylvain Gugger committed
2013

2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
    def get_memory_footprint(self, return_buffers=True):
        r"""
        Get the memory footprint of a model. This will return the memory footprint of the current model in bytes.
        Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the
        PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2

        Arguments:
            return_buffers (`bool`, *optional*, defaults to `True`):
                Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers
                are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch
                norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2
        """
        mem = sum([param.nelement() * param.element_size() for param in self.parameters()])
        if return_buffers:
            mem_bufs = sum([buf.nelement() * buf.element_size() for buf in self.buffers()])
            mem = mem + mem_bufs
        return mem

2032
    @wraps(torch.nn.Module.cuda)
2033
2034
    def cuda(self, *args, **kwargs):
        # Checks if the model has been loaded in 8-bit
Marc Sun's avatar
Marc Sun committed
2035
        if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
2036
2037
2038
2039
2040
2041
2042
            raise ValueError(
                "Calling `cuda()` is not supported for `4-bit` or `8-bit` quantized models. Please use the model as it is, since the"
                " model has already been set to the correct devices and casted to the correct `dtype`."
            )
        else:
            return super().cuda(*args, **kwargs)

2043
    @wraps(torch.nn.Module.to)
2044
2045
    def to(self, *args, **kwargs):
        # Checks if the model has been loaded in 8-bit
Marc Sun's avatar
Marc Sun committed
2046
        if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
2047
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2048
                "`.to` is not supported for `4-bit` or `8-bit` bitsandbytes models. Please use the model as it is, since the"
2049
2050
2051
2052
2053
2054
                " model has already been set to the correct devices and casted to the correct `dtype`."
            )
        else:
            return super().to(*args, **kwargs)

    def half(self, *args):
Marc Sun's avatar
Marc Sun committed
2055
        # Checks if the model is quantized
2056
        if getattr(self, "is_quantized", False):
2057
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2058
                "`.half()` is not supported for quantized model. Please use the model as it is, since the"
2059
2060
2061
2062
2063
2064
                " model has already been casted to the correct `dtype`."
            )
        else:
            return super().half(*args)

    def float(self, *args):
Marc Sun's avatar
Marc Sun committed
2065
        # Checks if the model is quantized
2066
        if getattr(self, "is_quantized", False):
2067
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2068
                "`.float()` is not supported for quantized model. Please use the model as it is, since the"
2069
2070
2071
2072
2073
                " model has already been casted to the correct `dtype`."
            )
        else:
            return super().float(*args)

2074
    @classmethod
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
        *model_args,
        config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
        cache_dir: Optional[Union[str, os.PathLike]] = None,
        ignore_mismatched_sizes: bool = False,
        force_download: bool = False,
        local_files_only: bool = False,
        token: Optional[Union[str, bool]] = None,
        revision: str = "main",
        use_safetensors: bool = None,
        **kwargs,
    ):
2089
2090
        r"""
        Instantiate a pretrained pytorch model from a pre-trained model configuration.
2091

Sylvain Gugger's avatar
Sylvain Gugger committed
2092
2093
        The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
        the model, you should first set it back in training mode with `model.train()`.
2094

2095
        The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come
2096
2097
        pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning
        task.
2098

2099
        The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those
2100
        weights are discarded.
2101

2102
        Parameters:
2103
            pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
2104
2105
                Can be either:

2106
                    - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
Sylvain Gugger's avatar
Sylvain Gugger committed
2107
2108
                      Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
                      user or organization name, like `dbmdz/bert-base-german-cased`.
2109
2110
2111
                    - A path to a *directory* containing model weights saved using
                      [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
                    - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
Sylvain Gugger's avatar
Sylvain Gugger committed
2112
2113
2114
                      this case, `from_tf` should be set to `True` and a configuration object should be provided as
                      `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
                      PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
2115
                    - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g,
Sylvain Gugger's avatar
Sylvain Gugger committed
2116
2117
                      `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to
                      `True`.
2118
2119
2120
2121
2122
                    - `None` if you are both providing the configuration and state dictionary (resp. with keyword
                      arguments `config` and `state_dict`).
            model_args (sequence of positional arguments, *optional*):
                All remaining positional arguments will be passed to the underlying model's `__init__` method.
            config (`Union[PretrainedConfig, str, os.PathLike]`, *optional*):
2123
2124
                Can be either:

2125
2126
                    - an instance of a class derived from [`PretrainedConfig`],
                    - a string or path valid as input to [`~PretrainedConfig.from_pretrained`].
2127

2128
                Configuration for the model to use instead of an automatically loaded configuration. Configuration can
2129
2130
                be automatically loaded when:

2131
                    - The model is a model provided by the library (loaded with the *model id* string of a pretrained
2132
                      model).
Sylvain Gugger's avatar
Sylvain Gugger committed
2133
2134
                    - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
                      save directory.
2135
2136
2137
                    - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
                      configuration JSON file named *config.json* is found in the directory.
            state_dict (`Dict[str, torch.Tensor]`, *optional*):
2138
2139
2140
                A state dictionary to use instead of a state dictionary loaded from saved weights file.

                This option can be used if you want to create a model from a pretrained configuration but load your own
Sylvain Gugger's avatar
Sylvain Gugger committed
2141
                weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and
2142
2143
                [`~PreTrainedModel.from_pretrained`] is not a simpler option.
            cache_dir (`Union[str, os.PathLike]`, *optional*):
2144
2145
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
2146
            from_tf (`bool`, *optional*, defaults to `False`):
2147
                Load the model weights from a TensorFlow checkpoint save file (see docstring of
2148
2149
                `pretrained_model_name_or_path` argument).
            from_flax (`bool`, *optional*, defaults to `False`):
2150
                Load the model weights from a Flax checkpoint save file (see docstring of
2151
2152
                `pretrained_model_name_or_path` argument).
            ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`):
2153
2154
2155
                Whether or not to raise an error if some of the weights from the checkpoint do not have the same size
                as the weights of the model (if for instance, you are instantiating a model with 10 labels from a
                checkpoint with 3 labels).
2156
            force_download (`bool`, *optional*, defaults to `False`):
2157
2158
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
2159
            resume_download (`bool`, *optional*, defaults to `False`):
2160
2161
                Whether or not to delete incompletely received files. Will attempt to resume the download if such a
                file exists.
2162
            proxies (`Dict[str, str]`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
2163
2164
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
2165
            output_loading_info(`bool`, *optional*, defaults to `False`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2166
                Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
2167
            local_files_only(`bool`, *optional*, defaults to `False`):
Stas Bekman's avatar
Stas Bekman committed
2168
                Whether or not to only look at local files (i.e., do not try to download the model).
2169
            token (`str` or `bool`, *optional*):
2170
2171
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
2172
            revision (`str`, *optional*, defaults to `"main"`):
Julien Chaumond's avatar
Julien Chaumond committed
2173
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
2174
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
Julien Chaumond's avatar
Julien Chaumond committed
2175
                identifier allowed by git.
2176
2177
2178
2179
2180
2181
2182

                <Tip>

                To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>".

                </Tip>

2183
            mirror (`str`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
2184
2185
2186
                Mirror source to accelerate downloads in China. If you are from China and have an accessibility
                problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety.
                Please refer to the mirror site for more information.
2187
            _fast_init(`bool`, *optional*, defaults to `True`):
2188
2189
                Whether or not to disable fast initialization.

2190
2191
                <Tip warning={true}>

Sylvain Gugger's avatar
Sylvain Gugger committed
2192
2193
2194
                One should only disable *_fast_init* to ensure backwards compatibility with `transformers.__version__ <
                4.6.0` for seeded model initialization. This argument will be removed at the next major version. See
                [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information.
2195

2196
                </Tip>
2197

2198
2199
2200
            > Parameters for big model inference

            low_cpu_mem_usage(`bool`, *optional*):
2201
2202
2203
                Tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
                This is an experimental feature and a subject to change at any moment.
            torch_dtype (`str` or `torch.dtype`, *optional*):
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
                Override the default `torch.dtype` and load the model under a specific `dtype`. The different options
                are:

                1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified
                  `dtype`, ignoring the model's `config.torch_dtype` if one exists. If not specified
                  - the model will get loaded in `torch.float` (fp32).

                2. `"auto"` - A `torch_dtype` entry in the `config.json` file of the model will be
                  attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in
                  the checkpoint that's of a floating point type and use that as `dtype`. This will load the model
                  using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how
                  the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32.

                <Tip>

                For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or
                reach out to the authors and ask them to add this information to the model's card and to insert the
                `torch_dtype` entry in `config.json` on the hub.

                </Tip>

2225
            device_map (`str` or `Dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*):
2226
2227
                A map that specifies where each submodule should go. It doesn't need to be refined to each
                parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the
2228
2229
2230
                same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank
                like `1`) on which the model will be allocated, the device map will map the entire model to this
                device. Passing `device_map = 0` means put the whole model on GPU 0.
2231

2232
2233
                To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For
                more information about each option see [designing a device
Patrick von Platen's avatar
Patrick von Platen committed
2234
                map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
2235
2236
2237
            max_memory (`Dict`, *optional*):
                A dictionary device identifier to maximum memory. Will default to the maximum memory available for each
                GPU and the available CPU RAM if unset.
2238
2239
            offload_folder (`str` or `os.PathLike`, *optional*):
                If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
2240
            offload_state_dict (`bool`, *optional*):
2241
                If `True`, will temporarily offload the CPU state dict to the hard drive to avoid getting out of CPU
2242
2243
                RAM if the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to
                `True` when there is some disk offload.
2244
2245
            load_in_8bit (`bool`, *optional*, defaults to `False`):
                If `True`, will convert the loaded model into mixed-8bit quantized model. To use this feature please
2246
2247
2248
2249
                install `bitsandbytes` (`pip install -U bitsandbytes`).
            load_in_4bit (`bool`, *optional*, defaults to `False`):
                If `True`, will convert the loaded model into 4bit precision quantized model. To use this feature
                install the latest version of `bitsandbytes` (`pip install -U bitsandbytes`).
Marc Sun's avatar
Marc Sun committed
2250
2251
2252
            quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*):
                A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g
                bitsandbytes, gptq)
2253
2254
2255
            subfolder (`str`, *optional*, defaults to `""`):
                In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
                specify the folder name here.
2256
2257
2258
            variant (`str`, *optional*):
                If specified load weights from `variant` filename, *e.g.* pytorch_model.<variant>.bin. `variant` is
                ignored when using `from_tf` or `from_flax`.
2259
2260
2261
            use_safetensors (`bool`, *optional*, defaults to `None`):
                Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors`
                is not installed, it will be set to `False`.
2262

2263
            kwargs (remaining dictionary of keyword arguments, *optional*):
2264
                Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
2265
                `output_attentions=True`). Behaves differently depending on whether a `config` is provided or
2266
2267
                automatically loaded:

2268
2269
                    - If a configuration is provided with `config`, `**kwargs` will be directly passed to the
                      underlying model's `__init__` method (we assume all relevant updates to the configuration have
2270
                      already been done)
2271
                    - If a configuration is not provided, `kwargs` will be first passed to the configuration class
Sylvain Gugger's avatar
Sylvain Gugger committed
2272
2273
2274
2275
                      initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
                      corresponds to a configuration attribute will be used to override said attribute with the
                      supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
                      will be passed to the underlying model's `__init__` function.
2276
2277
2278

        <Tip>

Sylvain Gugger's avatar
Sylvain Gugger committed
2279
2280
        Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to
        use this method in a firewalled environment.
2281
2282
2283
2284
2285
2286
2287

        </Tip>

        Examples:

        ```python
        >>> from transformers import BertConfig, BertModel
Sylvain Gugger's avatar
Sylvain Gugger committed
2288

2289
        >>> # Download model and configuration from huggingface.co and cache.
Sylvain Gugger's avatar
Sylvain Gugger committed
2290
        >>> model = BertModel.from_pretrained("bert-base-uncased")
2291
        >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable).
Sylvain Gugger's avatar
Sylvain Gugger committed
2292
        >>> model = BertModel.from_pretrained("./test/saved_model/")
2293
        >>> # Update configuration during loading.
Sylvain Gugger's avatar
Sylvain Gugger committed
2294
        >>> model = BertModel.from_pretrained("bert-base-uncased", output_attentions=True)
2295
2296
        >>> assert model.config.output_attentions == True
        >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable).
Sylvain Gugger's avatar
Sylvain Gugger committed
2297
2298
        >>> config = BertConfig.from_json_file("./tf_model/my_tf_model_config.json")
        >>> model = BertModel.from_pretrained("./tf_model/my_tf_checkpoint.ckpt.index", from_tf=True, config=config)
2299
        >>> # Loading from a Flax checkpoint file instead of a PyTorch model (slower)
Sylvain Gugger's avatar
Sylvain Gugger committed
2300
        >>> model = BertModel.from_pretrained("bert-base-uncased", from_flax=True)
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
        ```

        * `low_cpu_mem_usage` algorithm:

        This is an experimental function that loads the model using ~1x model size CPU memory

        Here is how it works:

        1. save which state_dict keys we have
        2. drop state_dict before the model is created, since the latter takes 1x model size CPU memory
        3. after the model has been instantiated switch to the meta device all params/buffers that
        are going to be replaced from the loaded state_dict
        4. load state_dict 2nd time
        5. replace the params/buffers from the state_dict

        Currently, it can't handle deepspeed ZeRO stage 3 and ignores loading errors

        """
2319
2320
        state_dict = kwargs.pop("state_dict", None)
        from_tf = kwargs.pop("from_tf", False)
2321
        from_flax = kwargs.pop("from_flax", False)
2322
2323
2324
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        output_loading_info = kwargs.pop("output_loading_info", False)
2325
        use_auth_token = kwargs.pop("use_auth_token", None)
2326
        trust_remote_code = kwargs.pop("trust_remote_code", None)
Sylvain Gugger's avatar
Sylvain Gugger committed
2327
        _ = kwargs.pop("mirror", None)
2328
2329
        from_pipeline = kwargs.pop("_from_pipeline", None)
        from_auto_class = kwargs.pop("_from_auto", False)
2330
        _fast_init = kwargs.pop("_fast_init", True)
2331
        torch_dtype = kwargs.pop("torch_dtype", None)
2332
2333
        low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", None)
        device_map = kwargs.pop("device_map", None)
2334
        max_memory = kwargs.pop("max_memory", None)
2335
        offload_folder = kwargs.pop("offload_folder", None)
2336
2337
        offload_state_dict = kwargs.pop("offload_state_dict", False)
        load_in_8bit = kwargs.pop("load_in_8bit", False)
2338
        load_in_4bit = kwargs.pop("load_in_4bit", False)
2339
        quantization_config = kwargs.pop("quantization_config", None)
2340
        subfolder = kwargs.pop("subfolder", "")
2341
        commit_hash = kwargs.pop("_commit_hash", None)
2342
        variant = kwargs.pop("variant", None)
2343
2344
        _adapter_model_path = kwargs.pop("_adapter_model_path", None)
        adapter_name = kwargs.pop("adapter_name", "default")
2345

2346
2347
2348
        if is_fsdp_enabled():
            low_cpu_mem_usage = True

2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        if use_safetensors is None and not is_safetensors_available():
            use_safetensors = False
2361

2362
        if is_bitsandbytes_available():
2363
            is_8bit_serializable = version.parse(importlib.metadata.version("bitsandbytes")) > version.parse("0.37.2")
2364
2365
2366
        else:
            is_8bit_serializable = False

2367
2368
2369
2370
2371
        if trust_remote_code is True:
            logger.warning(
                "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is"
                " ignored."
            )
2372

2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
        if commit_hash is None:
            if not isinstance(config, PretrainedConfig):
                # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible
                resolved_config_file = cached_file(
                    pretrained_model_name_or_path,
                    CONFIG_NAME,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
                    token=token,
                    revision=revision,
                    subfolder=subfolder,
                    _raise_exceptions_for_missing_entries=False,
                    _raise_exceptions_for_connection_errors=False,
                )
                commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
            else:
                commit_hash = getattr(config, "_commit_hash", None)

2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
        if is_peft_available():
            if _adapter_model_path is None:
                _adapter_model_path = find_adapter_config_file(
                    pretrained_model_name_or_path,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
                    token=token,
                    revision=revision,
                    subfolder=subfolder,
                    _commit_hash=commit_hash,
                )
            if _adapter_model_path is not None and os.path.isfile(_adapter_model_path):
                with open(_adapter_model_path, "r", encoding="utf-8"):
                    _adapter_model_path = pretrained_model_name_or_path
2411

2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
        # change device_map into a map if we passed an int, a str or a torch.device
        if isinstance(device_map, torch.device):
            device_map = {"": device_map}
        elif isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
            try:
                device_map = {"": torch.device(device_map)}
            except RuntimeError:
                raise ValueError(
                    "When passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or "
                    f"'auto', 'balanced', 'balanced_low_0', 'sequential' but found {device_map}."
                )
        elif isinstance(device_map, int):
            if device_map < 0:
                raise ValueError(
                    "You can't pass device_map as a negative int. If you want to put the model on the cpu, pass device_map = 'cpu' "
                )
            else:
                device_map = {"": device_map}

2431
2432
2433
2434
2435
2436
2437
        if device_map is not None:
            if low_cpu_mem_usage is None:
                low_cpu_mem_usage = True
            elif not low_cpu_mem_usage:
                raise ValueError("Passing along a `device_map` requires `low_cpu_mem_usage=True`")

        if low_cpu_mem_usage:
2438
            if device_map is not None:
2439
2440
                # The max memory utils require PyTorch >= 1.10 to have torch.cuda.mem_get_info.
                require_version_core("torch>=1.10")
2441
2442
2443
2444
2445
2446
2447
2448
2449

            if is_deepspeed_zero3_enabled():
                raise ValueError(
                    "DeepSpeed Zero-3 is not compatible with `low_cpu_mem_usage=True` or with passing a `device_map`."
                )
            elif not is_accelerate_available():
                raise ImportError(
                    "Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`"
                )
2450

Marc Sun's avatar
Marc Sun committed
2451
2452
2453
2454
2455
2456
2457
2458
        quantization_method_from_args = None
        if quantization_config is not None:
            quantization_method_from_args = getattr(
                quantization_config, "quant_method", QuantizationMethod.BITS_AND_BYTES
            )

        if quantization_config is None and (load_in_8bit or load_in_4bit):
            quantization_method_from_args = QuantizationMethod.BITS_AND_BYTES
2459
            quantization_config, kwargs = BitsAndBytesConfig.from_dict(
2460
2461
2462
                config_dict={"load_in_8bit": load_in_8bit, "load_in_4bit": load_in_4bit},
                return_unused_kwargs=True,
                **kwargs,
2463
            )
Marc Sun's avatar
Marc Sun committed
2464
        elif quantization_method_from_args == QuantizationMethod.BITS_AND_BYTES:
2465
            load_in_8bit = quantization_config.load_in_8bit
2466
            load_in_4bit = quantization_config.load_in_4bit
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477

            quantization_config_kwargs = {
                k: v for k, v in kwargs.items() if k in inspect.signature(BitsAndBytesConfig).parameters
            }

            if len(quantization_config_kwargs) > 0:
                raise ValueError(
                    "You can't pass `load_in_8bit` or any other `BitsAndBytesConfig` argument as a kwarg when passing "
                    "`quantization_config` argument at the same time."
                )

2478
        if load_in_8bit or load_in_4bit:
2479
2480
2481
2482
2483
2484
            if not (is_accelerate_available() and is_bitsandbytes_available()):
                raise ImportError(
                    "Using `load_in_8bit=True` requires Accelerate: `pip install accelerate` and the latest version of"
                    " bitsandbytes `pip install -i https://test.pypi.org/simple/ bitsandbytes` or"
                    " pip install bitsandbytes` "
                )
2485
2486

            if torch_dtype is None:
2487
                # We force the `dtype` to be float16, this is a requirement from `bitsandbytes`
2488
                logger.info(
2489
                    f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to "
2490
                    "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. "
2491
2492
                    "Pass your own torch_dtype to specify the dtype of the remaining non-linear layers or pass"
                    " torch_dtype=torch.float16 to remove this warning."
2493
                )
2494
                torch_dtype = torch.float16
2495

2496
            if device_map is None:
2497
2498
2499
2500
2501
2502
2503
2504
                if torch.cuda.is_available():
                    device_map = {"": torch.cuda.current_device()}
                else:
                    raise RuntimeError("No GPU found. A GPU is needed for quantization.")
                logger.info(
                    "The device_map was not initialized."
                    "Setting device_map to {'':torch.cuda.current_device()}."
                    "If you want to use the model for inference, please set device_map ='auto' "
2505
                )
2506
2507
2508
                if low_cpu_mem_usage is None:
                    low_cpu_mem_usage = True

2509
2510
            if from_tf or from_flax:
                raise ValueError(
2511
                    "Converting into 4-bit or 8-bit weights from tf/flax weights is currently not supported, please make"
2512
2513
2514
                    " sure the weights are in PyTorch format."
                )

2515
        from_pt = not (from_tf | from_flax)
2516
2517
2518
2519

        user_agent = {"file_type": "model", "framework": "pytorch", "from_auto_class": from_auto_class}
        if from_pipeline is not None:
            user_agent["using_pipeline"] = from_pipeline
thomwolf's avatar
thomwolf committed
2520

2521
2522
2523
2524
        if is_offline_mode() and not local_files_only:
            logger.info("Offline mode: forcing local_files_only=True")
            local_files_only = True

2525
2526
2527
        # Load config if we don't provide a configuration
        if not isinstance(config, PretrainedConfig):
            config_path = config if config is not None else pretrained_model_name_or_path
2528
            config, model_kwargs = cls.config_class.from_pretrained(
2529
2530
2531
                config_path,
                cache_dir=cache_dir,
                return_unused_kwargs=True,
2532
                force_download=force_download,
2533
                resume_download=resume_download,
2534
                proxies=proxies,
2535
                local_files_only=local_files_only,
2536
                token=token,
Julien Chaumond's avatar
Julien Chaumond committed
2537
                revision=revision,
2538
                subfolder=subfolder,
2539
2540
                _from_auto=from_auto_class,
                _from_pipeline=from_pipeline,
2541
                **kwargs,
2542
2543
2544
            )
        else:
            model_kwargs = kwargs
2545

Marc Sun's avatar
Marc Sun committed
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
        quantizer = None
        quantization_method_from_config = None
        if hasattr(config, "quantization_config"):
            quantization_method_from_config = config.quantization_config.get(
                "quant_method", QuantizationMethod.BITS_AND_BYTES
            )

        if quantization_method_from_config == QuantizationMethod.GPTQ and quantization_method_from_args is not None:
            loading_attr_dict = quantization_config.get_loading_attributes()
            for attr, val in loading_attr_dict.items():
                config.quantization_config[attr] = val
            quantization_method_from_args = None
            logger.warning(
                "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a "
                "`quantization_config` attribute and has already quantized weights. However, loading attributes"
                " (e.g. disable_exllama, use_cuda_fp16) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored."
            )
        if (
            quantization_method_from_args == QuantizationMethod.GPTQ
            or quantization_method_from_config == QuantizationMethod.GPTQ
        ):
            if not torch.cuda.is_available():
                raise RuntimeError("GPU is required to quantize or run quantize model.")
            elif not (is_optimum_available() and is_auto_gptq_available()):
                raise ImportError(
                    "Loading GPTQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'"
                )
            else:
                # Need to protect the import
                from optimum.gptq import GPTQQuantizer
            if quantization_method_from_config == QuantizationMethod.GPTQ:
                quantization_config = GPTQConfig.from_dict(config.quantization_config)
                config.quantization_config = quantization_config
            logger.info(
                f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to "
                "requirements of `auto-gptq` to enable model quantization "
            )
            torch_dtype = torch.float16
            quantizer = GPTQQuantizer.from_dict(quantization_config.to_dict())

        if (
            is_8bit_serializable
            and quantization_method_from_args == QuantizationMethod.BITS_AND_BYTES
            and load_in_8bit
        ):
            if quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES:
2592
2593
2594
2595
2596
2597
                logger.warning(
                    "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a"
                    " `quantization_config` attribute. The `quantization_config` attribute will be overwritten with the"
                    " one you passed to `from_pretrained`."
                )
            config.quantization_config = quantization_config
Marc Sun's avatar
Marc Sun committed
2598
2599
2600
2601
2602
        elif (
            is_8bit_serializable
            and not load_in_8bit
            and quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES
        ):
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
            quantization_config = config.quantization_config
            if isinstance(quantization_config, dict):
                quantization_config = BitsAndBytesConfig.from_dict(quantization_config, return_unused_kwargs=False)
            elif isinstance(quantization_config, BitsAndBytesConfig):
                pass
            else:
                raise ValueError(
                    f"Invalid type for `quantization_config`: {type(quantization_config)}. Should be a `dict` or a"
                    " `BitsAndBytesConfig` instance."
                )

            load_in_8bit = quantization_config.load_in_8bit

            if load_in_8bit:
2617
2618
                if torch_dtype is None:
                    torch_dtype = torch.float16
2619
                if device_map is None:
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
                    if torch.cuda.is_available():
                        device_map = {"": torch.cuda.current_device()}
                    else:
                        raise RuntimeError("No GPU found. A GPU is needed for quantization.")
                    logger.info(
                        "The device_map was not initialized."
                        "Setting device_map to {'':torch.cuda.current_device()}."
                        "If you want to use the model for inference, please set device_map ='auto' "
                    )
                    if low_cpu_mem_usage is None:
                        low_cpu_mem_usage = True
2631

Marc Sun's avatar
Marc Sun committed
2632
2633
2634
2635
2636
        elif (
            not is_8bit_serializable
            and not load_in_8bit
            and quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES
        ):
2637
2638
2639
2640
2641
2642
            logger.warning(
                "Detected the presence of a `quantization_config` attribute in the model's configuration but you don't have the correct"
                " `bitsandbytes` version to support int8 serialization. Please install the latest version of `bitsandbytes` with "
                " `pip install --upgrade bitsandbytes`."
            )

Sylvain Gugger's avatar
Sylvain Gugger committed
2643
2644
2645
2646
        # This variable will flag if we're loading a sharded checkpoint. In this case the archive file is just the
        # index of the files.
        is_sharded = False
        sharded_metadata = None
thomwolf's avatar
thomwolf committed
2647
        # Load model
Yih-Dar's avatar
Yih-Dar committed
2648
2649
        loading_info = None

2650
2651
2652
2653
        # Keep in fp32 modules
        keep_in_fp32_modules = None
        use_keep_in_fp32_modules = False

thomwolf's avatar
thomwolf committed
2654
        if pretrained_model_name_or_path is not None:
2655
            pretrained_model_name_or_path = str(pretrained_model_name_or_path)
2656
2657
            is_local = os.path.isdir(pretrained_model_name_or_path)
            if is_local:
2658
2659
2660
                if from_tf and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                ):
2661
                    # Load from a TF 1.0 checkpoint in priority if from_tf
2662
2663
2664
2665
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                elif from_tf and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)
                ):
2666
                    # Load from a TF 2.0 checkpoint in priority if from_tf
2667
2668
2669
2670
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)
                elif from_flax and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
                ):
2671
                    # Load from a Flax checkpoint in priority if from_flax
2672
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
2673
                elif use_safetensors is not False and os.path.isfile(
2674
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant))
2675
2676
                ):
                    # Load from a safetensors checkpoint
2677
2678
2679
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant)
                    )
2680
                elif use_safetensors is not False and os.path.isfile(
2681
2682
2683
                    os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)
                    )
2684
2685
                ):
                    # Load from a sharded safetensors checkpoint
2686
2687
2688
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)
                    )
2689
                    is_sharded = True
2690
2691
2692
                elif os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_NAME, variant))
                ):
thomwolf's avatar
thomwolf committed
2693
                    # Load from a PyTorch checkpoint
2694
2695
2696
2697
2698
2699
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_NAME, variant)
                    )
                elif os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_INDEX_NAME, variant))
                ):
Sylvain Gugger's avatar
Sylvain Gugger committed
2700
                    # Load from a sharded PyTorch checkpoint
2701
2702
2703
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_INDEX_NAME, variant)
                    )
Sylvain Gugger's avatar
Sylvain Gugger committed
2704
                    is_sharded = True
2705
2706
                # At this stage we don't have a weight file so we will raise an error.
                elif os.path.isfile(
2707
2708
                    os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                ) or os.path.isfile(os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)):
2709
                    raise EnvironmentError(
2710
2711
2712
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path} but there is a file for TensorFlow weights. Use"
                        " `from_tf=True` to load this model from those weights."
2713
                    )
2714
                elif os.path.isfile(os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)):
2715
                    raise EnvironmentError(
2716
2717
2718
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path} but there is a file for Flax weights. Use `from_flax=True`"
                        " to load this model from those weights."
2719
                    )
2720
2721
2722
2723
2724
                elif use_safetensors:
                    raise EnvironmentError(
                        f"Error no file named {_add_variant(SAFE_WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path}."
                    )
thomwolf's avatar
thomwolf committed
2725
                else:
2726
                    raise EnvironmentError(
2727
2728
2729
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)}, {TF2_WEIGHTS_NAME},"
                        f" {TF_WEIGHTS_NAME + '.index'} or {FLAX_WEIGHTS_NAME} found in directory"
                        f" {pretrained_model_name_or_path}."
2730
                    )
2731
            elif os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path)):
2732
                archive_file = pretrained_model_name_or_path
2733
                is_local = True
2734
            elif os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path + ".index")):
2735
2736
2737
2738
2739
                if not from_tf:
                    raise ValueError(
                        f"We found a TensorFlow checkpoint at {pretrained_model_name_or_path + '.index'}, please set "
                        "from_tf to True to load from this checkpoint."
                    )
2740
                archive_file = os.path.join(subfolder, pretrained_model_name_or_path + ".index")
2741
                is_local = True
2742
            elif is_remote_url(pretrained_model_name_or_path):
2743
                filename = pretrained_model_name_or_path
2744
                resolved_archive_file = download_url(pretrained_model_name_or_path)
2745
            else:
2746
2747
2748
2749
2750
                # set correct filename
                if from_tf:
                    filename = TF2_WEIGHTS_NAME
                elif from_flax:
                    filename = FLAX_WEIGHTS_NAME
2751
                elif use_safetensors is not False:
2752
                    filename = _add_variant(SAFE_WEIGHTS_NAME, variant)
2753
                else:
2754
                    filename = _add_variant(WEIGHTS_NAME, variant)
2755

2756
2757
                try:
                    # Load from URL or cache if already cached
2758
2759
2760
2761
2762
2763
                    cached_file_kwargs = {
                        "cache_dir": cache_dir,
                        "force_download": force_download,
                        "proxies": proxies,
                        "resume_download": resume_download,
                        "local_files_only": local_files_only,
2764
                        "token": token,
2765
2766
2767
2768
2769
2770
                        "user_agent": user_agent,
                        "revision": revision,
                        "subfolder": subfolder,
                        "_raise_exceptions_for_missing_entries": False,
                        "_commit_hash": commit_hash,
                    }
2771
                    resolved_archive_file = cached_file(pretrained_model_name_or_path, filename, **cached_file_kwargs)
2772

2773
                    # Since we set _raise_exceptions_for_missing_entries=False, we don't get an exception but a None
2774
                    # result when internet is up, the repo and revision exist, but the file does not.
2775
                    if resolved_archive_file is None and filename == _add_variant(SAFE_WEIGHTS_NAME, variant):
2776
2777
                        # Maybe the checkpoint is sharded, we try to grab the index name in this case.
                        resolved_archive_file = cached_file(
2778
2779
2780
                            pretrained_model_name_or_path,
                            _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant),
                            **cached_file_kwargs,
2781
2782
2783
                        )
                        if resolved_archive_file is not None:
                            is_sharded = True
2784
2785
2786
2787
                        elif use_safetensors:
                            raise EnvironmentError(
                                f" {_add_variant(SAFE_WEIGHTS_NAME, variant)} or {_add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)} and thus cannot be loaded with `safetensors`. Please make sure that the model has been saved with `safe_serialization=True` or do not set `use_safetensors=True`."
                            )
2788
2789
                        else:
                            # This repo has no safetensors file of any kind, we switch to PyTorch.
2790
                            filename = _add_variant(WEIGHTS_NAME, variant)
2791
                            resolved_archive_file = cached_file(
2792
                                pretrained_model_name_or_path, filename, **cached_file_kwargs
2793
                            )
2794
                    if resolved_archive_file is None and filename == _add_variant(WEIGHTS_NAME, variant):
Sylvain Gugger's avatar
Sylvain Gugger committed
2795
                        # Maybe the checkpoint is sharded, we try to grab the index name in this case.
2796
                        resolved_archive_file = cached_file(
2797
2798
2799
                            pretrained_model_name_or_path,
                            _add_variant(WEIGHTS_INDEX_NAME, variant),
                            **cached_file_kwargs,
2800
                        )
2801
2802
2803
                        if resolved_archive_file is not None:
                            is_sharded = True
                    if resolved_archive_file is None:
Sylvain Gugger's avatar
Sylvain Gugger committed
2804
2805
2806
2807
2808
                        # Otherwise, maybe there is a TF or Flax model file.  We try those to give a helpful error
                        # message.
                        has_file_kwargs = {
                            "revision": revision,
                            "proxies": proxies,
2809
                            "token": token,
Sylvain Gugger's avatar
Sylvain Gugger committed
2810
2811
2812
                        }
                        if has_file(pretrained_model_name_or_path, TF2_WEIGHTS_NAME, **has_file_kwargs):
                            raise EnvironmentError(
Sylvain Gugger's avatar
Sylvain Gugger committed
2813
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
2814
2815
                                f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file for TensorFlow weights."
                                " Use `from_tf=True` to load this model from those weights."
Sylvain Gugger's avatar
Sylvain Gugger committed
2816
2817
2818
                            )
                        elif has_file(pretrained_model_name_or_path, FLAX_WEIGHTS_NAME, **has_file_kwargs):
                            raise EnvironmentError(
Sylvain Gugger's avatar
Sylvain Gugger committed
2819
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
                                f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file for Flax weights. Use"
                                " `from_flax=True` to load this model from those weights."
                            )
                        elif variant is not None and has_file(
                            pretrained_model_name_or_path, WEIGHTS_NAME, **has_file_kwargs
                        ):
                            raise EnvironmentError(
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
                                f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file without the variant"
                                f" {variant}. Use `variant=None` to load this model from those weights."
Sylvain Gugger's avatar
Sylvain Gugger committed
2830
2831
2832
                            )
                        else:
                            raise EnvironmentError(
2833
2834
2835
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
                                f" {_add_variant(WEIGHTS_NAME, variant)}, {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME} or"
                                f" {FLAX_WEIGHTS_NAME}."
Sylvain Gugger's avatar
Sylvain Gugger committed
2836
                            )
2837
2838
2839
2840
2841
2842
                except EnvironmentError:
                    # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted
                    # to the original exception.
                    raise
                except Exception:
                    # For any other exception, we throw a generic error.
2843
                    raise EnvironmentError(
2844
2845
2846
                        f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it"
                        " from 'https://huggingface.co/models', make sure you don't have a local directory with the"
                        f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
2847
2848
                        f" directory containing a file named {_add_variant(WEIGHTS_NAME, variant)},"
                        f" {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME} or {FLAX_WEIGHTS_NAME}."
2849
                    )
2850

2851
            if is_local:
2852
                logger.info(f"loading weights file {archive_file}")
2853
                resolved_archive_file = archive_file
2854
            else:
2855
                logger.info(f"loading weights file {filename} from cache at {resolved_archive_file}")
2856
        else:
thomwolf's avatar
thomwolf committed
2857
            resolved_archive_file = None
2858

Sylvain Gugger's avatar
Sylvain Gugger committed
2859
2860
        # We'll need to download and cache each checkpoint shard if the checkpoint is sharded.
        if is_sharded:
2861
            # rsolved_archive_file becomes a list of files that point to the different checkpoint shards in this case.
Sylvain Gugger's avatar
Sylvain Gugger committed
2862
2863
2864
2865
2866
2867
2868
2869
            resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
                pretrained_model_name_or_path,
                resolved_archive_file,
                cache_dir=cache_dir,
                force_download=force_download,
                proxies=proxies,
                resume_download=resume_download,
                local_files_only=local_files_only,
2870
                token=token,
Sylvain Gugger's avatar
Sylvain Gugger committed
2871
2872
                user_agent=user_agent,
                revision=revision,
2873
                subfolder=subfolder,
2874
                _commit_hash=commit_hash,
Sylvain Gugger's avatar
Sylvain Gugger committed
2875
2876
            )

2877
2878
        # load pt weights early so that we know which dtype to init the model under
        if from_pt:
2879
            if not is_sharded and state_dict is None:
Sylvain Gugger's avatar
Sylvain Gugger committed
2880
2881
                # Time to load the checkpoint
                state_dict = load_state_dict(resolved_archive_file)
2882

2883
2884
2885
            # set dtype to instantiate the model under:
            # 1. If torch_dtype is not None, we use that dtype
            # 2. If torch_dtype is "auto", we auto-detect dtype from the loaded state_dict, by checking its first
2886
            #    weights entry that is of a floating type - we assume all floating dtype weights are of the same dtype
2887
2888
            # we also may have config.torch_dtype available, but we won't rely on it till v5
            dtype_orig = None
2889

2890
2891
2892
            if torch_dtype is not None:
                if isinstance(torch_dtype, str):
                    if torch_dtype == "auto":
2893
2894
2895
                        if hasattr(config, "torch_dtype") and config.torch_dtype is not None:
                            torch_dtype = config.torch_dtype
                            logger.info(f"Will use torch_dtype={torch_dtype} as defined in model's config object")
Sylvain Gugger's avatar
Sylvain Gugger committed
2896
                        else:
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
                            if is_sharded and "dtype" in sharded_metadata:
                                torch_dtype = sharded_metadata["dtype"]
                            elif not is_sharded:
                                torch_dtype = get_state_dict_dtype(state_dict)
                            else:
                                one_state_dict = load_state_dict(resolved_archive_file[0])
                                torch_dtype = get_state_dict_dtype(one_state_dict)
                                del one_state_dict  # free CPU memory
                            logger.info(
                                "Since the `torch_dtype` attribute can't be found in model's config object, "
                                "will use torch_dtype={torch_dtype} as derived from model's weights"
                            )
2909
2910
                    else:
                        raise ValueError(
2911
                            f'`torch_dtype` can be either `torch.dtype` or `"auto"`, but received {torch_dtype}'
2912
2913
2914
                        )
                dtype_orig = cls._set_default_torch_dtype(torch_dtype)

2915
2916
            # Check if `_keep_in_fp32_modules` is not None
            use_keep_in_fp32_modules = (
2917
2918
2919
                (cls._keep_in_fp32_modules is not None)
                and is_accelerate_available()
                and (torch_dtype == torch.float16 or load_in_4bit or load_in_8bit)
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
            )
            if (
                (cls._keep_in_fp32_modules is not None)
                and not is_accelerate_available()
                and torch_dtype == torch.float16
            ):
                logger.warning(
                    "For stability purposes, it is recommended to have accelerate installed when using this model in"
                    " torch.float16, please install it with `pip install accelerate`"
                )

2931
2932
2933
            if is_sharded:
                loaded_state_dict_keys = sharded_metadata["all_checkpoint_keys"]
            else:
2934
                loaded_state_dict_keys = list(state_dict.keys())
2935
            if low_cpu_mem_usage or use_keep_in_fp32_modules:
2936
                state_dict = None
2937

2938
2939
        config.name_or_path = pretrained_model_name_or_path

2940
        # Instantiate model.
2941
2942
        init_contexts = [no_init_weights(_enable=_fast_init)]

2943
2944
2945
2946
        if is_deepspeed_zero3_enabled():
            import deepspeed

            logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
2947
            init_contexts = [deepspeed.zero.Init(config_dict_or_path=deepspeed_config())] + init_contexts
2948
        elif load_in_8bit or load_in_4bit or low_cpu_mem_usage:
2949
2950
2951
2952
2953
            init_contexts.append(init_empty_weights())

        with ContextManagers(init_contexts):
            model = cls(config, *model_args, **model_kwargs)

2954
2955
2956
2957
2958
2959
2960
        # Check first if we are `from_pt`
        if use_keep_in_fp32_modules:
            low_cpu_mem_usage = True
            keep_in_fp32_modules = model._keep_in_fp32_modules
        else:
            keep_in_fp32_modules = []

2961
2962
        if load_in_8bit or load_in_4bit:
            from .utils.bitsandbytes import get_keys_to_not_convert, replace_with_bnb_linear
2963

2964
            llm_int8_skip_modules = quantization_config.llm_int8_skip_modules
2965
            load_in_8bit_fp32_cpu_offload = quantization_config.llm_int8_enable_fp32_cpu_offload
2966
2967
2968
2969
            if load_in_8bit:
                logger.info("Detected 8-bit loading: activating 8-bit loading for this model")
            else:
                logger.info("Detected 4-bit loading: activating 4-bit loading for this model")
2970

2971
            # We keep some modules such as the lm_head in their original dtype for numerical stability reasons
2972
            if llm_int8_skip_modules is None:
2973
2974
                modules_to_not_convert = get_keys_to_not_convert(model)
            else:
2975
                modules_to_not_convert = llm_int8_skip_modules
2976
2977
2978
2979
2980
2981

            if not isinstance(modules_to_not_convert, list):
                modules_to_not_convert = [modules_to_not_convert]

            modules_to_not_convert.extend(keep_in_fp32_modules)

2982
2983
2984
2985
2986
2987
2988
            # Extend the modules to not convert to keys that are supposed to be offloaded to `cpu` or `disk`
            if isinstance(device_map, dict) and len(device_map.keys()) > 1:
                keys_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]]

                if len(keys_on_cpu) > 0 and not load_in_8bit_fp32_cpu_offload:
                    raise ValueError(
                        "If you want to offload some keys to `cpu` or `disk`, you need to set "
Younes Belkada's avatar
Younes Belkada committed
2989
                        "`llm_int8_enable_fp32_cpu_offload=True`. Note that these modules will not be "
2990
2991
2992
2993
2994
                        " converted to 8-bit but kept in 32-bit."
                    )

                modules_to_not_convert.extend(keys_on_cpu)

2995
            supports_4bit = version.parse(importlib.metadata.version("bitsandbytes")) >= version.parse("0.39.0")
2996
2997
2998
2999
3000
3001
3002
3003
3004

            if load_in_4bit and not supports_4bit:
                raise ValueError(
                    "You have a version of `bitsandbytes` that is not compatible with 4bit inference and training"
                    " make sure you have the latest version of `bitsandbytes` installed"
                )

            model = replace_with_bnb_linear(
                model, modules_to_not_convert=modules_to_not_convert, quantization_config=quantization_config
3005
            )
3006
            # training in 8-bit is only available in 0.37.0+
3007
            model._is_quantized_training_enabled = version.parse(
3008
                importlib.metadata.version("bitsandbytes")
3009
            ) >= version.parse("0.37.0")
3010

3011
3012
3013
            model.config.quantization_config = quantization_config
            model.is_8bit_serializable = is_8bit_serializable

3014
3015
3016
        if load_in_8bit and torch_dtype is None:
            logger.warning(
                "You are loading your model in 8bit but you did not specify a `torch_dtype` attribute."
3017
3018
                "All non-linear modules will be loaded in full precision."
                " If you want to load the other modules in other precision, please specify a `torch_dtype` attribute."
3019
            )
Marc Sun's avatar
Marc Sun committed
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
        if quantization_method_from_config == QuantizationMethod.GPTQ:
            model = quantizer.convert_model(model)
            model._is_quantized_training_enabled = True

        if quantization_method_from_config is not None:
            model.quantization_method = quantization_method_from_config
        elif quantization_method_from_args is not None:
            model.quantization_method = quantization_method_from_args
        if hasattr(model, "quantization_method"):
            model.is_quantized = True
3030

3031
        if isinstance(device_map, str):
3032
            special_dtypes = {}
3033
            if load_in_8bit or load_in_4bit:
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
                special_dtypes.update(
                    {
                        name: torch_dtype
                        for name, _ in model.named_parameters()
                        if any(m in name for m in modules_to_not_convert)
                    }
                )

            special_dtypes.update(
                {
                    name: torch.float32
                    for name, _ in model.named_parameters()
                    if any(m in name for m in keep_in_fp32_modules)
                }
            )

3050
3051
3052
            target_dtype = torch_dtype

            if load_in_4bit:
3053
                if version.parse(importlib.metadata.version("accelerate")) > version.parse("0.19.0"):
3054
3055
3056
3057
3058
3059
3060
                    from accelerate.utils import CustomDtype

                    target_dtype = CustomDtype.INT4
                else:
                    raise ValueError(
                        "You are using `device_map='auto'` on a 4bit loaded version of the model. To automatically compute"
                        " the appropriate device map, you should upgrade your `accelerate` library,"
3061
                        "`pip install --upgrade accelerate` or install it from source to support fp4 auto device map"
3062
3063
3064
3065
3066
                        "calculation. You may encounter unexpected behavior, or pass your own device map"
                    )
            elif load_in_8bit:
                target_dtype = torch.int8

3067
            if model._no_split_modules is None:
3068
3069
3070
3071
                raise ValueError(
                    f"{model.__class__.__name__} does not support `device_map='{device_map}'`. To implement support, the model"
                    "class needs to implement the `_no_split_modules` attribute."
                )
3072
            no_split_modules = model._no_split_modules
3073
3074
3075
3076
3077
            if device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
                raise ValueError(
                    "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or "
                    "'sequential'."
                )
3078

3079
            device_map_kwargs = {"no_split_module_classes": no_split_modules}
3080
            if "special_dtypes" in inspect.signature(infer_auto_device_map).parameters:
3081
                device_map_kwargs["special_dtypes"] = special_dtypes
3082
            elif len(special_dtypes) > 0:
3083
                logger.warning(
3084
3085
3086
                    "This model has some weights that should be kept in higher precision, you need to upgrade "
                    "`accelerate` to properly deal with them (`pip install --upgrade accelerate`)."
                )
3087
            if device_map != "sequential":
3088
3089
                max_memory = get_balanced_memory(
                    model,
3090
                    dtype=target_dtype,
3091
                    low_zero=(device_map == "balanced_low_0"),
3092
                    max_memory=max_memory,
3093
                    **device_map_kwargs,
3094
                )
3095
            device_map_kwargs["max_memory"] = max_memory
3096
3097
            # Make sure tied weights are tied before creating the device map.
            model.tie_weights()
3098
            device_map = infer_auto_device_map(model, dtype=target_dtype, **device_map_kwargs)
3099

3100
            if load_in_8bit or load_in_4bit:
3101
                # The LM head / tied weights or any last module can stay on disk / CPU
3102
                device_map_without_lm_head = {
3103
                    key: device_map[key] for key in device_map.keys() if key not in modules_to_not_convert
3104
3105
                }
                if "cpu" in device_map_without_lm_head.values() or "disk" in device_map_without_lm_head.values():
3106
3107
3108
                    raise ValueError(
                        """
                        Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit
3109
3110
3111
3112
3113
                        the quantized model. If you want to dispatch the model on the CPU or the disk while keeping
                        these modules in 32-bit, you need to set `load_in_8bit_fp32_cpu_offload=True` and pass a custom
                        `device_map` to `from_pretrained`. Check
                        https://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-cpu-and-gpu
                        for more details.
3114
3115
                        """
                    )
3116
3117
                del device_map_without_lm_head

3118
3119
3120
3121
        elif device_map is not None:
            model.tie_weights()
            tied_params = find_tied_parameters(model)
            # check if we don't have tied param in different devices
3122
            check_tied_parameters_on_same_device(tied_params, device_map)
3123

3124
        if from_tf:
3125
            if resolved_archive_file.endswith(".index"):
3126
3127
3128
3129
3130
                # Load from a TensorFlow 1.X checkpoint - provided by original authors
                model = cls.load_tf_weights(model, config, resolved_archive_file[:-6])  # Remove the '.index'
            else:
                # Load from our TensorFlow 2.0 checkpoints
                try:
3131
                    from .modeling_tf_pytorch_utils import load_tf2_checkpoint_in_pytorch_model
3132

Yih-Dar's avatar
Yih-Dar committed
3133
3134
3135
                    model, loading_info = load_tf2_checkpoint_in_pytorch_model(
                        model, resolved_archive_file, allow_missing_keys=True, output_loading_info=True
                    )
3136
                except ImportError:
3137
                    logger.error(
Sylvain Gugger's avatar
Sylvain Gugger committed
3138
3139
3140
                        "Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed."
                        " Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation"
                        " instructions."
3141
                    )
3142
                    raise
3143
3144
3145
3146
3147
3148
3149
        elif from_flax:
            try:
                from .modeling_flax_pytorch_utils import load_flax_checkpoint_in_pytorch_model

                model = load_flax_checkpoint_in_pytorch_model(model, resolved_archive_file)
            except ImportError:
                logger.error(
Sylvain Gugger's avatar
Sylvain Gugger committed
3150
3151
3152
                    "Loading a Flax model in PyTorch, requires both PyTorch and Flax to be installed. Please see"
                    " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for"
                    " installation instructions."
3153
3154
                )
                raise
3155
        elif from_pt:
3156
3157
3158
3159
            # restore default dtype
            if dtype_orig is not None:
                torch.set_default_dtype(dtype_orig)

Sylvain Gugger's avatar
Sylvain Gugger committed
3160
3161
3162
3163
3164
3165
3166
3167
            (
                model,
                missing_keys,
                unexpected_keys,
                mismatched_keys,
                offload_index,
                error_msgs,
            ) = cls._load_pretrained_model(
3168
3169
3170
3171
3172
3173
3174
3175
3176
                model,
                state_dict,
                loaded_state_dict_keys,  # XXX: rename?
                resolved_archive_file,
                pretrained_model_name_or_path,
                ignore_mismatched_sizes=ignore_mismatched_sizes,
                sharded_metadata=sharded_metadata,
                _fast_init=_fast_init,
                low_cpu_mem_usage=low_cpu_mem_usage,
3177
3178
3179
3180
                device_map=device_map,
                offload_folder=offload_folder,
                offload_state_dict=offload_state_dict,
                dtype=torch_dtype,
Marc Sun's avatar
Marc Sun committed
3181
                is_quantized=(getattr(model, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES),
3182
                keep_in_fp32_modules=keep_in_fp32_modules,
3183
            )
3184

3185
        model.is_loaded_in_4bit = load_in_4bit
Younes Belkada's avatar
Younes Belkada committed
3186
        model.is_loaded_in_8bit = load_in_8bit
3187

3188
3189
        # make sure token embedding weights are still tied if needed
        model.tie_weights()
3190

3191
        # Set model in evaluation mode to deactivate DropOut modules by default
3192
3193
        model.eval()

3194
        # If it is a model with generation capabilities, attempt to load the generation config
3195
        if model.can_generate() and pretrained_model_name_or_path is not None:
3196
3197
3198
3199
3200
3201
3202
3203
            try:
                model.generation_config = GenerationConfig.from_pretrained(
                    pretrained_model_name_or_path,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
3204
                    token=token,
3205
3206
3207
3208
3209
3210
                    revision=revision,
                    subfolder=subfolder,
                    _from_auto=from_auto_class,
                    _from_pipeline=from_pipeline,
                    **kwargs,
                )
3211
            except OSError:
3212
3213
3214
3215
3216
                logger.info(
                    "Generation config file not found, using a generation config created from the model config."
                )
                pass

3217
3218
        # Dispatch model with hooks on all devices if necessary
        if device_map is not None:
3219
3220
3221
3222
3223
            device_map_kwargs = {
                "device_map": device_map,
                "offload_dir": offload_folder,
                "offload_index": offload_index,
            }
3224
            if "skip_keys" in inspect.signature(dispatch_model).parameters:
3225
3226
                device_map_kwargs["skip_keys"] = model._skip_keys_device_placement
            dispatch_model(model, **device_map_kwargs)
3227

Marc Sun's avatar
Marc Sun committed
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
        if quantization_method_from_args == QuantizationMethod.GPTQ:
            if quantization_config.tokenizer is None:
                quantization_config.tokenizer = pretrained_model_name_or_path
            if cls.main_input_name != "input_ids":
                raise RuntimeError("We can only quantize pure text model.")
            quantizer.quantize_model(model, quantization_config.tokenizer)
            model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict())
            model._is_quantized_training_enabled = True
        if quantization_method_from_config == QuantizationMethod.GPTQ:
            model = quantizer.post_init_model(model)

3239
        if _adapter_model_path is not None:
3240
            model.load_adapter(
3241
                _adapter_model_path,
3242
3243
3244
3245
3246
                adapter_name=adapter_name,
                revision=revision,
                token=token,
            )

thomwolf's avatar
thomwolf committed
3247
        if output_loading_info:
Yih-Dar's avatar
Yih-Dar committed
3248
3249
3250
3251
3252
3253
3254
            if loading_info is None:
                loading_info = {
                    "missing_keys": missing_keys,
                    "unexpected_keys": unexpected_keys,
                    "mismatched_keys": mismatched_keys,
                    "error_msgs": error_msgs,
                }
thomwolf's avatar
thomwolf committed
3255
3256
            return model, loading_info

3257
3258
        return model

3259
    @classmethod
Sylvain Gugger's avatar
Sylvain Gugger committed
3260
3261
3262
3263
    def _load_pretrained_model(
        cls,
        model,
        state_dict,
3264
        loaded_keys,
Sylvain Gugger's avatar
Sylvain Gugger committed
3265
3266
3267
3268
3269
        resolved_archive_file,
        pretrained_model_name_or_path,
        ignore_mismatched_sizes=False,
        sharded_metadata=None,
        _fast_init=True,
3270
        low_cpu_mem_usage=False,
3271
3272
        device_map=None,
        offload_folder=None,
3273
        offload_state_dict=None,
3274
        dtype=None,
3275
        is_quantized=False,
3276
        keep_in_fp32_modules=None,
3277
    ):
Sylvain Gugger's avatar
Sylvain Gugger committed
3278
        is_safetensors = False
3279
3280
        if is_quantized:
            from .utils.bitsandbytes import set_module_quantized_tensor_to_device
3281

Sylvain Gugger's avatar
Sylvain Gugger committed
3282
        if device_map is not None and "disk" in device_map.values():
Sylvain Gugger's avatar
Sylvain Gugger committed
3283
3284
3285
3286
3287
            archive_file = (
                resolved_archive_file[0] if isinstance(resolved_archive_file, (list, tuple)) else resolved_archive_file
            )
            is_safetensors = archive_file.endswith(".safetensors")
            if offload_folder is None and not is_safetensors:
Sylvain Gugger's avatar
Sylvain Gugger committed
3288
3289
                raise ValueError(
                    "The current `device_map` had weights offloaded to the disk. Please provide an `offload_folder`"
Sylvain Gugger's avatar
Sylvain Gugger committed
3290
3291
                    " for them. Alternatively, make sure you have `safetensors` installed if the model you are using"
                    " offers the weights in this format."
Sylvain Gugger's avatar
Sylvain Gugger committed
3292
                )
Sylvain Gugger's avatar
Sylvain Gugger committed
3293
3294
            if offload_folder is not None:
                os.makedirs(offload_folder, exist_ok=True)
3295
3296
3297
            if offload_state_dict is None:
                offload_state_dict = True

3298
        is_sharded_safetensors = is_safetensors and sharded_metadata is not None
Patrick von Platen's avatar
Patrick von Platen committed
3299
3300
3301
3302

        # tie the model weights before retrieving the state_dict
        model.tie_weights()

3303
        # Retrieve missing & unexpected_keys
3304
3305
        model_state_dict = model.state_dict()
        expected_keys = list(model_state_dict.keys())
3306
3307
        prefix = model.base_model_prefix

Sylvain Gugger's avatar
Sylvain Gugger committed
3308
3309
3310
3311
3312
3313
3314
        def _fix_key(key):
            if "beta" in key:
                return key.replace("beta", "bias")
            if "gamma" in key:
                return key.replace("gamma", "weight")
            return key

3315
        original_loaded_keys = loaded_keys
Sylvain Gugger's avatar
Sylvain Gugger committed
3316
3317
        loaded_keys = [_fix_key(key) for key in loaded_keys]

3318
3319
3320
3321
3322
3323
        if len(prefix) > 0:
            has_prefix_module = any(s.startswith(prefix) for s in loaded_keys)
            expects_prefix_module = any(s.startswith(prefix) for s in expected_keys)
        else:
            has_prefix_module = False
            expects_prefix_module = False
Patrick von Platen's avatar
Patrick von Platen committed
3324
3325
3326

        # key re-naming operations are never done on the keys
        # that are loaded, but always on the keys of the newly initialized model
3327
3328
        remove_prefix_from_model = not has_prefix_module and expects_prefix_module
        add_prefix_to_model = has_prefix_module and not expects_prefix_module
3329

3330
        if remove_prefix_from_model:
3331
3332
3333
            _prefix = f"{prefix}."
            expected_keys_not_prefixed = [s for s in expected_keys if not s.startswith(_prefix)]
            expected_keys = [s[len(_prefix) :] if s.startswith(_prefix) else s for s in expected_keys]
3334
        elif add_prefix_to_model:
3335
3336
3337
            expected_keys = [".".join([prefix, s]) for s in expected_keys]

        missing_keys = list(set(expected_keys) - set(loaded_keys))
Sylvain Gugger's avatar
Sylvain Gugger committed
3338
3339
3340
3341
3342
3343
3344
3345
3346
        unexpected_keys = set(loaded_keys) - set(expected_keys)
        # Remove nonpersistent buffers from unexpected keys: they are not in the state dict but will be in the model
        # buffers
        model_buffers = {n for n, _ in model.named_buffers()}
        if remove_prefix_from_model:
            model_buffers = {key[len(_prefix) :] if key.startswith(_prefix) else key for key in model_buffers}
        elif add_prefix_to_model:
            model_buffers = {".".join([prefix, key]) for key in model_buffers}
        unexpected_keys = list(unexpected_keys - model_buffers)
3347

3348
3349
        model.tie_weights()
        if device_map is None and not is_fsdp_enabled():
3350
3351
3352
3353
            ptrs = collections.defaultdict(list)
            for name, tensor in model.state_dict().items():
                id_tensor = id_tensor_storage(tensor)
                ptrs[id_tensor].append(name)
Sylvain Gugger's avatar
Sylvain Gugger committed
3354

3355
3356
3357
3358
3359
            # These are all the pointers of shared tensors.
            tied_params = [names for _, names in ptrs.items() if len(names) > 1]
        else:
            # id function doesn't work for meta tensor so we need this function
            tied_params = find_tied_parameters(model)
Sylvain Gugger's avatar
Sylvain Gugger committed
3360
3361

        for group in tied_params:
Sylvain Gugger's avatar
Sylvain Gugger committed
3362
3363
3364
3365
            if remove_prefix_from_model:
                group = [key[len(_prefix) :] if key.startswith(_prefix) else key for key in group]
            elif add_prefix_to_model:
                group = [".".join([prefix, key]) for key in group]
Sylvain Gugger's avatar
Sylvain Gugger committed
3366
3367
3368
            missing_in_group = [k for k in missing_keys if k in group]
            if len(missing_in_group) > 0 and len(missing_in_group) < len(group):
                missing_keys = [k for k in missing_keys if k not in missing_in_group]
3369

3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
        # Some models may have keys that are not in the state by design, removing them before needlessly warning
        # the user.
        if cls._keys_to_ignore_on_load_missing is not None:
            for pat in cls._keys_to_ignore_on_load_missing:
                missing_keys = [k for k in missing_keys if re.search(pat, k) is None]

        if cls._keys_to_ignore_on_load_unexpected is not None:
            for pat in cls._keys_to_ignore_on_load_unexpected:
                unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]

3380
3381
3382
3383
        # retrieve weights on meta device and put them back on CPU.
        # This is not ideal in terms of memory, but if we don't do that not, we can't initialize them in the next step
        if low_cpu_mem_usage:
            for key in missing_keys:
Susnato Dhar's avatar
Susnato Dhar committed
3384
3385
                if key in list(model_state_dict.keys()):
                    key = key
3386
3387
                elif f"{prefix}.{key}" in list(model_state_dict.keys()):
                    key = f"{prefix}.{key}"
Susnato Dhar's avatar
Susnato Dhar committed
3388
                elif key.startswith(prefix) and ".".join(key.split(".")[1:]) in list(model_state_dict.keys()):
3389
3390
                    key = ".".join(key.split(".")[1:])
                param = model_state_dict[key]
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400

                # upcast in fp32 if any
                target_dtype = dtype
                if (
                    keep_in_fp32_modules is not None
                    and dtype == torch.float16
                    and any(module_to_keep_in_fp32 in key for module_to_keep_in_fp32 in keep_in_fp32_modules)
                ):
                    target_dtype = torch.float32

3401
                if param.device == torch.device("meta"):
3402
                    if not (is_quantized):
3403
                        set_module_tensor_to_device(model, key, "cpu", torch.empty(*param.size(), dtype=target_dtype))
3404
                    else:
3405
                        set_module_quantized_tensor_to_device(
3406
3407
                            model, key, "cpu", torch.empty(*param.size(), dtype=target_dtype)
                        )
3408
3409

        # retrieve unintialized modules and initialize before maybe overriding that with the pretrained weights.
3410
        if _fast_init:
3411
3412
3413
3414
3415
3416
3417
3418
3419
            if remove_prefix_from_model:
                _loaded_keys = [f"{prefix}.{k}" for k in loaded_keys]
            elif add_prefix_to_model:
                _loaded_keys = [k[len(prefix) + 1 :] for k in loaded_keys]
            else:
                _loaded_keys = loaded_keys
            set_initialized_submodules(model, _loaded_keys)
            # This will only initialize submodules that are not marked as initialized by the line above.
            model.apply(model._initialize_weights)
3420

3421
3422
3423
3424
3425
3426
        # Set some modules to fp32 if any
        if keep_in_fp32_modules is not None:
            for name, param in model.named_parameters():
                if any(module_to_keep_in_fp32 in name for module_to_keep_in_fp32 in keep_in_fp32_modules):
                    param = param.to(torch.float32)

3427
3428
3429
        # Make sure we are able to load base models as well as derived models (with heads)
        start_prefix = ""
        model_to_load = model
3430
        if len(cls.base_model_prefix) > 0 and not hasattr(model, cls.base_model_prefix) and has_prefix_module:
3431
            start_prefix = cls.base_model_prefix + "."
3432
        if len(cls.base_model_prefix) > 0 and hasattr(model, cls.base_model_prefix) and not has_prefix_module:
3433
            model_to_load = getattr(model, cls.base_model_prefix)
Sylvain Gugger's avatar
Sylvain Gugger committed
3434
3435
            base_model_expected_keys = list(model_to_load.state_dict().keys())
            if any(key in expected_keys_not_prefixed and key not in base_model_expected_keys for key in loaded_keys):
3436
                raise ValueError(
3437
                    "The state dictionary of the model you are trying to load is corrupted. Are you sure it was "
3438
3439
                    "properly saved?"
                )
3440
3441
            if device_map is not None:
                device_map = {k.replace(f"{cls.base_model_prefix}.", ""): v for k, v in device_map.items()}
3442

3443
3444
3445
3446
3447
3448
3449
3450
        def _find_mismatched_keys(
            state_dict,
            model_state_dict,
            loaded_keys,
            add_prefix_to_model,
            remove_prefix_from_model,
            ignore_mismatched_sizes,
        ):
Sylvain Gugger's avatar
Sylvain Gugger committed
3451
3452
3453
            mismatched_keys = []
            if ignore_mismatched_sizes:
                for checkpoint_key in loaded_keys:
3454
3455
3456
                    # If the checkpoint is sharded, we may not have the key here.
                    if checkpoint_key not in state_dict:
                        continue
Sylvain Gugger's avatar
Sylvain Gugger committed
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
                    model_key = checkpoint_key
                    if remove_prefix_from_model:
                        # The model key starts with `prefix` but `checkpoint_key` doesn't so we add it.
                        model_key = f"{prefix}.{checkpoint_key}"
                    elif add_prefix_to_model:
                        # The model key doesn't start with `prefix` but `checkpoint_key` does so we remove it.
                        model_key = ".".join(checkpoint_key.split(".")[1:])

                    if (
                        model_key in model_state_dict
                        and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape
                    ):
                        mismatched_keys.append(
                            (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape)
                        )
                        del state_dict[checkpoint_key]
3473
3474
            return mismatched_keys

3475
3476
3477
3478
        if resolved_archive_file is not None:
            folder = os.path.sep.join(resolved_archive_file[0].split(os.path.sep)[:-1])
        else:
            folder = None
Sylvain Gugger's avatar
Sylvain Gugger committed
3479
        if device_map is not None and is_safetensors:
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
            param_device_map = expand_device_map(device_map, original_loaded_keys)

            str_dtype = str(dtype).replace("torch.", "") if dtype is not None else "float32"
            if sharded_metadata is None:
                archive_file = (
                    resolved_archive_file[0]
                    if isinstance(resolved_archive_file, (list, tuple))
                    else resolved_archive_file
                )
                weight_map = {p: archive_file for p in original_loaded_keys}
            else:
                weight_map = {p: os.path.join(folder, f) for p, f in sharded_metadata["weight_map"].items()}
Sylvain Gugger's avatar
Sylvain Gugger committed
3492
            offload_index = {
3493
3494
                p: {"safetensors_file": f, "weight_name": p, "dtype": str_dtype}
                for p, f in weight_map.items()
Sylvain Gugger's avatar
Sylvain Gugger committed
3495
3496
3497
                if param_device_map[p] == "disk"
            }

3498
3499
3500
3501
3502
        if state_dict is not None:
            # Whole checkpoint
            mismatched_keys = _find_mismatched_keys(
                state_dict,
                model_state_dict,
3503
                original_loaded_keys,
3504
3505
3506
3507
                add_prefix_to_model,
                remove_prefix_from_model,
                ignore_mismatched_sizes,
            )
Sylvain Gugger's avatar
Sylvain Gugger committed
3508
            error_msgs = _load_state_dict_into_model(model_to_load, state_dict, start_prefix)
Sylvain Gugger's avatar
Sylvain Gugger committed
3509
            offload_index = None
Sylvain Gugger's avatar
Sylvain Gugger committed
3510
        else:
3511
3512
            # Sharded checkpoint or whole but low_cpu_mem_usage==True

Sylvain Gugger's avatar
Sylvain Gugger committed
3513
3514
3515
3516
3517
            # This should always be a list but, just to be sure.
            if not isinstance(resolved_archive_file, list):
                resolved_archive_file = [resolved_archive_file]

            error_msgs = []
3518
            mismatched_keys = []
Sylvain Gugger's avatar
Sylvain Gugger committed
3519
3520
            if not is_safetensors:
                offload_index = {} if device_map is not None and "disk" in device_map.values() else None
3521
3522
3523
3524
3525
3526
3527
            if offload_state_dict:
                state_dict_folder = tempfile.mkdtemp()
                state_dict_index = {}
            else:
                state_dict_folder = None
                state_dict_index = None

3528
            if is_sharded_safetensors:
Sylvain Gugger's avatar
Sylvain Gugger committed
3529
3530
3531
3532
3533
                disk_only_shard_files = get_disk_only_shard_files(device_map, sharded_metadata=sharded_metadata)
                disk_only_shard_files = [os.path.join(folder, f) for f in disk_only_shard_files]
            else:
                disk_only_shard_files = []

3534
3535
            if len(resolved_archive_file) > 1:
                resolved_archive_file = logging.tqdm(resolved_archive_file, desc="Loading checkpoint shards")
Sylvain Gugger's avatar
Sylvain Gugger committed
3536
            for shard_file in resolved_archive_file:
Sylvain Gugger's avatar
Sylvain Gugger committed
3537
3538
3539
                # Skip the load for shards that only contain disk-offloaded weights when using safetensors for the offload.
                if shard_file in disk_only_shard_files:
                    continue
Sylvain Gugger's avatar
Sylvain Gugger committed
3540
                state_dict = load_state_dict(shard_file)
3541

Sylvain Gugger's avatar
Sylvain Gugger committed
3542
3543
                # Mistmatched keys contains tuples key/shape1/shape2 of weights in the checkpoint that have a shape not
                # matching the weights in the model.
3544
3545
3546
                mismatched_keys += _find_mismatched_keys(
                    state_dict,
                    model_state_dict,
3547
                    original_loaded_keys,
3548
3549
3550
3551
                    add_prefix_to_model,
                    remove_prefix_from_model,
                    ignore_mismatched_sizes,
                )
3552
3553

                if low_cpu_mem_usage:
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
                    if not is_fsdp_enabled() or is_fsdp_enabled_and_dist_rank_0():
                        new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(
                            model_to_load,
                            state_dict,
                            loaded_keys,
                            start_prefix,
                            expected_keys,
                            device_map=device_map,
                            offload_folder=offload_folder,
                            offload_index=offload_index,
                            state_dict_folder=state_dict_folder,
                            state_dict_index=state_dict_index,
                            dtype=dtype,
                            is_quantized=is_quantized,
                            is_safetensors=is_safetensors,
                            keep_in_fp32_modules=keep_in_fp32_modules,
                        )
                        error_msgs += new_error_msgs
                    else:
                        for key, param in model_to_load.state_dict().items():
                            if param.device == torch.device("meta"):
                                if not (is_quantized):
                                    set_module_tensor_to_device(
                                        model, key, "cpu", torch.empty(*param.size(), dtype=dtype)
                                    )
                                else:
                                    set_module_quantized_tensor_to_device(
                                        model, key, "cpu", torch.empty(*param.size(), dtype=dtype)
                                    )
3583
3584
                else:
                    error_msgs += _load_state_dict_into_model(model_to_load, state_dict, start_prefix)
3585

3586
3587
3588
3589
                # force memory release
                del state_dict
                gc.collect()

3590
            if offload_index is not None and len(offload_index) > 0:
Sylvain Gugger's avatar
Sylvain Gugger committed
3591
3592
3593
                if model != model_to_load:
                    # We need to add the prefix of the base model
                    prefix = cls.base_model_prefix
Sylvain Gugger's avatar
Sylvain Gugger committed
3594
3595
3596
3597
3598
3599
                    if not is_safetensors:
                        for weight_name in offload_index:
                            shutil.move(
                                os.path.join(offload_folder, f"{weight_name}.dat"),
                                os.path.join(offload_folder, f"{prefix}.{weight_name}.dat"),
                            )
Sylvain Gugger's avatar
Sylvain Gugger committed
3600
                    offload_index = {f"{prefix}.{key}": value for key, value in offload_index.items()}
Sylvain Gugger's avatar
Sylvain Gugger committed
3601
3602
3603
                if not is_safetensors:
                    save_offload_index(offload_index, offload_folder)
                    offload_index = None
3604
3605
3606

            if offload_state_dict:
                # Load back temporarily offloaded state dict
3607
                load_offloaded_weights(model_to_load, state_dict_index, state_dict_folder)
3608
3609
                shutil.rmtree(state_dict_folder)

3610
3611
        if len(error_msgs) > 0:
            error_msg = "\n\t".join(error_msgs)
3612
3613
3614
3615
            if "size mismatch" in error_msg:
                error_msg += (
                    "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method."
                )
3616
3617
            raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}")

3618
        if is_quantized:
3619
3620
3621
            unexpected_keys = [elem for elem in unexpected_keys if "SCB" not in elem]
            missing_keys = [elem for elem in missing_keys if "SCB" not in elem]

3622
        if len(unexpected_keys) > 0:
Sylvain Gugger's avatar
Sylvain Gugger committed
3623
            archs = [] if model.config.architectures is None else model.config.architectures
3624
            warner = logger.warning if model.__class__.__name__ in archs else logger.info
Sylvain Gugger's avatar
Sylvain Gugger committed
3625
            warner(
Sylvain Gugger's avatar
Sylvain Gugger committed
3626
3627
3628
3629
3630
3631
3632
                f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"
                f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"
                f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or"
                " with another architecture (e.g. initializing a BertForSequenceClassification model from a"
                " BertForPreTraining model).\n- This IS NOT expected if you are initializing"
                f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly identical"
                " (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)."
3633
3634
3635
3636
3637
            )
        else:
            logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n")
        if len(missing_keys) > 0:
            logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
3638
3639
3640
                f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"
                " TRAIN this model on a down-stream task to be able to use it for predictions and inference."
3641
            )
3642
        elif len(mismatched_keys) == 0:
3643
            logger.info(
Sylvain Gugger's avatar
Sylvain Gugger committed
3644
3645
3646
3647
                f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the checkpoint"
                f" was trained on, you can already use {model.__class__.__name__} for predictions without further"
                " training."
3648
            )
3649
3650
3651
3652
3653
3654
3655
3656
        if len(mismatched_keys) > 0:
            mismatched_warning = "\n".join(
                [
                    f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated"
                    for key, shape1, shape2 in mismatched_keys
                ]
            )
            logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
3657
3658
3659
3660
                f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not"
                f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be able"
                " to use it for predictions and inference."
3661
            )
3662

Sylvain Gugger's avatar
Sylvain Gugger committed
3663
        return model, missing_keys, unexpected_keys, mismatched_keys, offload_index, error_msgs
3664
3665

    def retrieve_modules_from_names(self, names, add_prefix=False, remove_prefix=False):
3666
        module_keys = {".".join(key.split(".")[:-1]) for key in names}
3667

Patrick von Platen's avatar
Patrick von Platen committed
3668
3669
        # torch.nn.ParameterList is a special case where two parameter keywords
        # are appended to the module name, *e.g.* bert.special_embeddings.0
3670
        module_keys = module_keys.union(
3671
            {".".join(key.split(".")[:-2]) for key in names if len(key) > 0 and key[-1].isdigit()}
3672
        )
Patrick von Platen's avatar
Patrick von Platen committed
3673

3674
3675
3676
3677
        retrieved_modules = []
        # retrieve all modules that has at least one missing weight name
        for name, module in self.named_modules():
            if remove_prefix:
3678
3679
                _prefix = f"{self.base_model_prefix}."
                name = name[len(_prefix) :] if name.startswith(_prefix) else name
3680
            elif add_prefix:
Patrick von Platen's avatar
Patrick von Platen committed
3681
                name = ".".join([self.base_model_prefix, name]) if len(name) > 0 else self.base_model_prefix
3682
3683
3684
3685
3686
3687

            if name in module_keys:
                retrieved_modules.append(module)

        return retrieved_modules

3688
    @staticmethod
3689
    def _load_pretrained_model_low_mem(model, loaded_state_dict_keys, resolved_archive_file, start_prefix=""):
3690
3691
3692
        """
        This is an experimental function that loads the model using ~1.x model size CPU memory

3693
        Before you call it do:
3694

3695
        1. save which state_dict keys are available
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
        2. drop state_dict before model is created, since the latter takes 1x model size memory

        Here then we continue:

        3. switch to the meta device all params/buffers that are going to be replaced from the loaded state_dict
        4. load state_dict 2nd time
        5. replace the params/buffers from the state_dict

        Currently, it doesn't handle missing_keys, unexpected_keys, mismatched_keys. It can't handle deepspeed.
        """

3707
3708
3709
3710
        _move_model_to_meta(model, loaded_state_dict_keys, start_prefix)
        state_dict = load_state_dict(resolved_archive_file)
        error_msgs = _load_state_dict_into_meta_model(model, state_dict, loaded_state_dict_keys, start_prefix)
        return error_msgs
3711

3712
3713
3714
3715
3716
3717
    @classmethod
    def register_for_auto_class(cls, auto_class="AutoModel"):
        """
        Register this class with a given auto class. This should only be used for custom models as the ones in the
        library are already mapped with an auto class.

3718
3719
3720
3721
3722
3723
        <Tip warning={true}>

        This API is experimental and may have some slight breaking changes in the next releases.

        </Tip>

3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
        Args:
            auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`):
                The auto class to register this new model with.
        """
        if not isinstance(auto_class, str):
            auto_class = auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(auto_module, auto_class):
            raise ValueError(f"{auto_class} is not a valid auto class.")

        cls._auto_class = auto_class

3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
    def to_bettertransformer(self) -> "PreTrainedModel":
        """
        Converts the model to use [PyTorch's native attention
        implementation](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html), integrated to
        Transformers through [Optimum library](https://huggingface.co/docs/optimum/bettertransformer/overview). Only a
        subset of all Transformers models are supported.

        PyTorch's attention fastpath allows to speed up inference through kernel fusions and the use of [nested
        tensors](https://pytorch.org/docs/stable/nested.html). Detailed benchmarks can be found in [this blog
        post](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2).

        Returns:
            [`PreTrainedModel`]: The model converted to BetterTransformer.
        """
        if not is_optimum_available():
            raise ImportError("The package `optimum` is required to use Better Transformer.")

        from optimum.version import __version__ as optimum_version

        if version.parse(optimum_version) < version.parse("1.7.0"):
            raise ImportError(
                f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
            )

        from optimum.bettertransformer import BetterTransformer

        return BetterTransformer.transform(self)

    def reverse_bettertransformer(self):
        """
        Reverts the transformation from [`~PreTrainedModel.to_bettertransformer`] so that the original modeling is
        used, for example in order to save the model.

        Returns:
            [`PreTrainedModel`]: The model converted back to the original modeling.
        """
        if not is_optimum_available():
            raise ImportError("The package `optimum` is required to use Better Transformer.")

        from optimum.version import __version__ as optimum_version

        if version.parse(optimum_version) < version.parse("1.7.0"):
            raise ImportError(
                f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
            )

        from optimum.bettertransformer import BetterTransformer

        return BetterTransformer.reverse(self)

3788
3789
3790
3791
    def warn_if_padding_and_no_attention_mask(self, input_ids, attention_mask):
        """
        Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given.
        """
3792
3793
3794
3795
3796

        # Skip the check during tracing.
        if is_torch_fx_proxy(input_ids) or torch.jit.is_tracing():
            return

3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
        if (attention_mask is not None) or (self.config.pad_token_id is None):
            return

        # Check only the first and last input IDs to reduce overhead.
        if self.config.pad_token_id in input_ids[:, [-1, 0]]:
            warn_string = (
                "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See "
                "https://huggingface.co/docs/transformers/troubleshooting"
                "#incorrect-output-when-padding-tokens-arent-masked."
            )

            # If the pad token is equal to either BOS, EOS, or SEP, we do not know whether the user should use an
            # attention_mask or not. In this case, we should still show a warning because this is a rare case.
            if (
                (self.config.bos_token_id is not None and self.config.bos_token_id == self.config.pad_token_id)
                or (self.config.eos_token_id is not None and self.config.eos_token_id == self.config.pad_token_id)
                or (self.config.sep_token_id is not None and self.config.sep_token_id == self.config.pad_token_id)
            ):
                warn_string += (
                    f"\nYou may ignore this warning if your `pad_token_id` ({self.config.pad_token_id}) is identical "
                    f"to the `bos_token_id` ({self.config.bos_token_id}), `eos_token_id` ({self.config.eos_token_id}), "
                    f"or the `sep_token_id` ({self.config.sep_token_id}), and your input is not padded."
                )

            logger.warning_once(warn_string)

thomwolf's avatar
thomwolf committed
3823

3824
PreTrainedModel.push_to_hub = copy_func(PreTrainedModel.push_to_hub)
3825
3826
3827
3828
if PreTrainedModel.push_to_hub.__doc__ is not None:
    PreTrainedModel.push_to_hub.__doc__ = PreTrainedModel.push_to_hub.__doc__.format(
        object="model", object_class="AutoModel", object_files="model file"
    )
3829
3830


thomwolf's avatar
thomwolf committed
3831
class PoolerStartLogits(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
3832
3833
    """
    Compute SQuAD start logits from sequence hidden states.
3834

Sylvain Gugger's avatar
Sylvain Gugger committed
3835
    Args:
3836
3837
        config ([`PretrainedConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
Sylvain Gugger's avatar
Sylvain Gugger committed
3838
3839
3840
    """

    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
3841
        super().__init__()
thomwolf's avatar
thomwolf committed
3842
3843
        self.dense = nn.Linear(config.hidden_size, 1)

Sylvain Gugger's avatar
Sylvain Gugger committed
3844
3845
3846
3847
3848
    def forward(
        self, hidden_states: torch.FloatTensor, p_mask: Optional[torch.FloatTensor] = None
    ) -> torch.FloatTensor:
        """
        Args:
3849
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
3850
                The final hidden states of the model.
3851
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3852
3853
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
Sylvain Gugger's avatar
Sylvain Gugger committed
3854
3855

        Returns:
3856
            `torch.FloatTensor`: The start logits for SQuAD.
thomwolf's avatar
thomwolf committed
3857
        """
thomwolf's avatar
thomwolf committed
3858
3859
3860
        x = self.dense(hidden_states).squeeze(-1)

        if p_mask is not None:
Lysandre Debut's avatar
Lysandre Debut committed
3861
            if get_parameter_dtype(self) == torch.float16:
3862
3863
3864
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
3865
3866
3867
3868
3869
3870

        return x


class PoolerEndLogits(nn.Module):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
3871
    Compute SQuAD end logits from sequence hidden states.
3872

Sylvain Gugger's avatar
Sylvain Gugger committed
3873
    Args:
3874
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
3875
3876
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
Sylvain Gugger's avatar
Sylvain Gugger committed
3877
3878
3879
    """

    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
3880
        super().__init__()
thomwolf's avatar
thomwolf committed
3881
3882
3883
3884
3885
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dense_1 = nn.Linear(config.hidden_size, 1)

Sylvain Gugger's avatar
Sylvain Gugger committed
3886
3887
3888
3889
3890
3891
3892
3893
3894
    def forward(
        self,
        hidden_states: torch.FloatTensor,
        start_states: Optional[torch.FloatTensor] = None,
        start_positions: Optional[torch.LongTensor] = None,
        p_mask: Optional[torch.FloatTensor] = None,
    ) -> torch.FloatTensor:
        """
        Args:
3895
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
3896
                The final hidden states of the model.
3897
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3898
                The hidden states of the first tokens for the labeled span.
3899
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3900
                The position of the first token for the labeled span.
3901
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3902
3903
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
Sylvain Gugger's avatar
Sylvain Gugger committed
3904

3905
        <Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
3906

Stas Bekman's avatar
Stas Bekman committed
3907
3908
        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.
3909
3910

        </Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
3911
3912

        Returns:
3913
            `torch.FloatTensor`: The end logits for SQuAD.
thomwolf's avatar
thomwolf committed
3914
        """
3915
3916
3917
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
3918
        if start_positions is not None:
3919
            slen, hsz = hidden_states.shape[-2:]
3920
3921
3922
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions)  # shape (bsz, 1, hsz)
            start_states = start_states.expand(-1, slen, -1)  # shape (bsz, slen, hsz)
thomwolf's avatar
thomwolf committed
3923
3924
3925
3926
3927
3928
3929

        x = self.dense_0(torch.cat([hidden_states, start_states], dim=-1))
        x = self.activation(x)
        x = self.LayerNorm(x)
        x = self.dense_1(x).squeeze(-1)

        if p_mask is not None:
Lysandre Debut's avatar
Lysandre Debut committed
3930
            if get_parameter_dtype(self) == torch.float16:
3931
3932
3933
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
3934
3935
3936
3937
3938

        return x


class PoolerAnswerClass(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
3939
3940
3941
3942
    """
    Compute SQuAD 2.0 answer class from classification and start tokens hidden states.

    Args:
3943
3944
        config ([`PretrainedConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
Sylvain Gugger's avatar
Sylvain Gugger committed
3945
    """
3946

thomwolf's avatar
thomwolf committed
3947
    def __init__(self, config):
Julien Chaumond's avatar
Julien Chaumond committed
3948
        super().__init__()
thomwolf's avatar
thomwolf committed
3949
3950
3951
3952
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.dense_1 = nn.Linear(config.hidden_size, 1, bias=False)

Sylvain Gugger's avatar
Sylvain Gugger committed
3953
3954
3955
3956
3957
3958
3959
    def forward(
        self,
        hidden_states: torch.FloatTensor,
        start_states: Optional[torch.FloatTensor] = None,
        start_positions: Optional[torch.LongTensor] = None,
        cls_index: Optional[torch.LongTensor] = None,
    ) -> torch.FloatTensor:
3960
3961
        """
        Args:
3962
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
3963
                The final hidden states of the model.
3964
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3965
                The hidden states of the first tokens for the labeled span.
3966
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3967
                The position of the first token for the labeled span.
3968
3969
3970
3971
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.

        <Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
3972

Stas Bekman's avatar
Stas Bekman committed
3973
3974
        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.
Sylvain Gugger's avatar
Sylvain Gugger committed
3975

3976
        </Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
3977
3978

        Returns:
3979
            `torch.FloatTensor`: The SQuAD 2.0 answer class.
thomwolf's avatar
thomwolf committed
3980
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
3981
        # No dependency on end_feature so that we can obtain one single `cls_logits` for each sample.
3982
        hsz = hidden_states.shape[-1]
3983
3984
3985
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
3986
        if start_positions is not None:
3987
3988
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
3989
3990

        if cls_index is not None:
3991
3992
            cls_index = cls_index[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            cls_token_state = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
3993
        else:
3994
            cls_token_state = hidden_states[:, -1, :]  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
3995
3996
3997
3998
3999
4000
4001
4002

        x = self.dense_0(torch.cat([start_states, cls_token_state], dim=-1))
        x = self.activation(x)
        x = self.dense_1(x).squeeze(-1)

        return x


4003
4004
4005
@dataclass
class SquadHeadOutput(ModelOutput):
    """
4006
    Base class for outputs of question answering models using a [`~modeling_utils.SQuADHead`].
4007
4008

    Args:
4009
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
Sylvain Gugger's avatar
Sylvain Gugger committed
4010
4011
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification
            losses.
4012
        start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
4013
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
4014
        start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
4015
            Indices for the top config.start_n_top start token possibilities (beam-search).
4016
4017
        end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
Sylvain Gugger's avatar
Sylvain Gugger committed
4018
            (beam-search).
4019
4020
4021
4022
        end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
        cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the `is_impossible` label of the answers.
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033

    """

    loss: Optional[torch.FloatTensor] = None
    start_top_log_probs: Optional[torch.FloatTensor] = None
    start_top_index: Optional[torch.LongTensor] = None
    end_top_log_probs: Optional[torch.FloatTensor] = None
    end_top_index: Optional[torch.LongTensor] = None
    cls_logits: Optional[torch.FloatTensor] = None


thomwolf's avatar
thomwolf committed
4034
class SQuADHead(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4035
4036
    r"""
    A SQuAD head inspired by XLNet.
4037

Sylvain Gugger's avatar
Sylvain Gugger committed
4038
    Args:
4039
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4040
4041
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
thomwolf's avatar
thomwolf committed
4042
    """
4043

thomwolf's avatar
thomwolf committed
4044
    def __init__(self, config):
Julien Chaumond's avatar
Julien Chaumond committed
4045
        super().__init__()
thomwolf's avatar
thomwolf committed
4046
4047
4048
4049
4050
4051
4052
        self.start_n_top = config.start_n_top
        self.end_n_top = config.end_n_top

        self.start_logits = PoolerStartLogits(config)
        self.end_logits = PoolerEndLogits(config)
        self.answer_class = PoolerAnswerClass(config)

Sylvain Gugger's avatar
Sylvain Gugger committed
4053
    @replace_return_docstrings(output_type=SquadHeadOutput, config_class=PretrainedConfig)
4054
    def forward(
4055
        self,
Sylvain Gugger's avatar
Sylvain Gugger committed
4056
4057
4058
4059
4060
4061
        hidden_states: torch.FloatTensor,
        start_positions: Optional[torch.LongTensor] = None,
        end_positions: Optional[torch.LongTensor] = None,
        cls_index: Optional[torch.LongTensor] = None,
        is_impossible: Optional[torch.LongTensor] = None,
        p_mask: Optional[torch.FloatTensor] = None,
4062
        return_dict: bool = False,
Sylvain Gugger's avatar
Sylvain Gugger committed
4063
4064
    ) -> Union[SquadHeadOutput, Tuple[torch.FloatTensor]]:
        """
Lysandre's avatar
Lysandre committed
4065
        Args:
4066
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Lysandre's avatar
Lysandre committed
4067
                Final hidden states of the model on the sequence tokens.
4068
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4069
                Positions of the first token for the labeled span.
4070
            end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4071
                Positions of the last token for the labeled span.
4072
4073
4074
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.
            is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4075
                Whether the question has a possible answer in the paragraph or not.
4076
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4077
4078
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
4079
            return_dict (`bool`, *optional*, defaults to `False`):
4080
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
Sylvain Gugger's avatar
Sylvain Gugger committed
4081

Lysandre's avatar
Lysandre committed
4082
        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
4083
        """
thomwolf's avatar
thomwolf committed
4084
        start_logits = self.start_logits(hidden_states, p_mask=p_mask)
thomwolf's avatar
thomwolf committed
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107

        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, let's remove the dimension added by batch splitting
            for x in (start_positions, end_positions, cls_index, is_impossible):
                if x is not None and x.dim() > 1:
                    x.squeeze_(-1)

            # during training, compute the end logits based on the ground truth of the start position
            end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask)

            loss_fct = CrossEntropyLoss()
            start_loss = loss_fct(start_logits, start_positions)
            end_loss = loss_fct(end_logits, end_positions)
            total_loss = (start_loss + end_loss) / 2

            if cls_index is not None and is_impossible is not None:
                # Predict answerability from the representation of CLS and START
                cls_logits = self.answer_class(hidden_states, start_positions=start_positions, cls_index=cls_index)
                loss_fct_cls = nn.BCEWithLogitsLoss()
                cls_loss = loss_fct_cls(cls_logits, is_impossible)

                # note(zhiliny): by default multiply the loss by 0.5 so that the scale is comparable to start_loss and end_loss
                total_loss += cls_loss * 0.5
4108

4109
            return SquadHeadOutput(loss=total_loss) if return_dict else (total_loss,)
thomwolf's avatar
thomwolf committed
4110
4111
4112
4113

        else:
            # during inference, compute the end logits based on beam search
            bsz, slen, hsz = hidden_states.size()
4114
            start_log_probs = nn.functional.softmax(start_logits, dim=-1)  # shape (bsz, slen)
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125

            start_top_log_probs, start_top_index = torch.topk(
                start_log_probs, self.start_n_top, dim=-1
            )  # shape (bsz, start_n_top)
            start_top_index_exp = start_top_index.unsqueeze(-1).expand(-1, -1, hsz)  # shape (bsz, start_n_top, hsz)
            start_states = torch.gather(hidden_states, -2, start_top_index_exp)  # shape (bsz, start_n_top, hsz)
            start_states = start_states.unsqueeze(1).expand(-1, slen, -1, -1)  # shape (bsz, slen, start_n_top, hsz)

            hidden_states_expanded = hidden_states.unsqueeze(2).expand_as(
                start_states
            )  # shape (bsz, slen, start_n_top, hsz)
thomwolf's avatar
thomwolf committed
4126
4127
            p_mask = p_mask.unsqueeze(-1) if p_mask is not None else None
            end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask)
4128
            end_log_probs = nn.functional.softmax(end_logits, dim=1)  # shape (bsz, slen, start_n_top)
thomwolf's avatar
thomwolf committed
4129

4130
4131
4132
            end_top_log_probs, end_top_index = torch.topk(
                end_log_probs, self.end_n_top, dim=1
            )  # shape (bsz, end_n_top, start_n_top)
thomwolf's avatar
thomwolf committed
4133
4134
4135
4136
4137
4138
            end_top_log_probs = end_top_log_probs.view(-1, self.start_n_top * self.end_n_top)
            end_top_index = end_top_index.view(-1, self.start_n_top * self.end_n_top)

            start_states = torch.einsum("blh,bl->bh", hidden_states, start_log_probs)
            cls_logits = self.answer_class(hidden_states, start_states=start_states, cls_index=cls_index)

4139
            if not return_dict:
4140
4141
4142
4143
4144
4145
4146
4147
4148
                return (start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits)
            else:
                return SquadHeadOutput(
                    start_top_log_probs=start_top_log_probs,
                    start_top_index=start_top_index,
                    end_top_log_probs=end_top_log_probs,
                    end_top_index=end_top_index,
                    cls_logits=cls_logits,
                )
thomwolf's avatar
thomwolf committed
4149
4150
4151


class SequenceSummary(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4152
4153
4154
4155
    r"""
    Compute a single vector summary of a sequence hidden states.

    Args:
4156
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4157
4158
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):
Sylvain Gugger's avatar
Sylvain Gugger committed
4159

4160
            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:
Sylvain Gugger's avatar
Sylvain Gugger committed
4161

4162
4163
4164
4165
4166
                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention
Sylvain Gugger's avatar
Sylvain Gugger committed
4167

4168
            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
Sylvain Gugger's avatar
Sylvain Gugger committed
4169
4170
4171
4172
4173
4174
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
thomwolf's avatar
thomwolf committed
4175
    """
4176

4177
    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
4178
        super().__init__()
thomwolf's avatar
thomwolf committed
4179

4180
        self.summary_type = getattr(config, "summary_type", "last")
4181
        if self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
4182
4183
4184
4185
4186
            # We should use a standard multi-head attention module with absolute positional embedding for that.
            # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276
            # We can probably just use the multi-head attention module of PyTorch >=1.1.0
            raise NotImplementedError

thomwolf's avatar
thomwolf committed
4187
        self.summary = Identity()
4188
4189
        if hasattr(config, "summary_use_proj") and config.summary_use_proj:
            if hasattr(config, "summary_proj_to_labels") and config.summary_proj_to_labels and config.num_labels > 0:
4190
                num_classes = config.num_labels
thomwolf's avatar
thomwolf committed
4191
4192
4193
4194
            else:
                num_classes = config.hidden_size
            self.summary = nn.Linear(config.hidden_size, num_classes)

4195
        activation_string = getattr(config, "summary_activation", None)
Lysandre's avatar
Lysandre committed
4196
        self.activation: Callable = get_activation(activation_string) if activation_string else Identity()
thomwolf's avatar
thomwolf committed
4197

thomwolf's avatar
thomwolf committed
4198
        self.first_dropout = Identity()
4199
        if hasattr(config, "summary_first_dropout") and config.summary_first_dropout > 0:
4200
4201
            self.first_dropout = nn.Dropout(config.summary_first_dropout)

thomwolf's avatar
thomwolf committed
4202
        self.last_dropout = Identity()
4203
        if hasattr(config, "summary_last_dropout") and config.summary_last_dropout > 0:
4204
            self.last_dropout = nn.Dropout(config.summary_last_dropout)
thomwolf's avatar
thomwolf committed
4205

Sylvain Gugger's avatar
Sylvain Gugger committed
4206
4207
4208
4209
4210
4211
4212
    def forward(
        self, hidden_states: torch.FloatTensor, cls_index: Optional[torch.LongTensor] = None
    ) -> torch.FloatTensor:
        """
        Compute a single vector summary of a sequence hidden states.

        Args:
4213
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4214
                The hidden states of the last layer.
4215
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4216
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.
Sylvain Gugger's avatar
Sylvain Gugger committed
4217
4218

        Returns:
4219
            `torch.FloatTensor`: The summary of the sequence hidden states.
thomwolf's avatar
thomwolf committed
4220
        """
4221
        if self.summary_type == "last":
thomwolf's avatar
thomwolf committed
4222
            output = hidden_states[:, -1]
4223
        elif self.summary_type == "first":
thomwolf's avatar
thomwolf committed
4224
            output = hidden_states[:, 0]
4225
        elif self.summary_type == "mean":
thomwolf's avatar
thomwolf committed
4226
            output = hidden_states.mean(dim=1)
4227
        elif self.summary_type == "cls_index":
thomwolf's avatar
thomwolf committed
4228
            if cls_index is None:
Lysandre's avatar
Lysandre committed
4229
4230
4231
4232
4233
                cls_index = torch.full_like(
                    hidden_states[..., :1, :],
                    hidden_states.shape[-2] - 1,
                    dtype=torch.long,
                )
thomwolf's avatar
thomwolf committed
4234
            else:
thomwolf's avatar
thomwolf committed
4235
                cls_index = cls_index.unsqueeze(-1).unsqueeze(-1)
4236
                cls_index = cls_index.expand((-1,) * (cls_index.dim() - 1) + (hidden_states.size(-1),))
thomwolf's avatar
thomwolf committed
4237
            # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states
4238
4239
            output = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, XX, hidden_size)
        elif self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
4240
4241
            raise NotImplementedError

4242
        output = self.first_dropout(output)
thomwolf's avatar
thomwolf committed
4243
4244
        output = self.summary(output)
        output = self.activation(output)
4245
        output = self.last_dropout(output)
thomwolf's avatar
thomwolf committed
4246
4247
4248
4249

        return output


4250
def unwrap_model(model: nn.Module) -> nn.Module:
4251
4252
4253
4254
    """
    Recursively unwraps a model from potential containers (as used in distributed training).

    Args:
4255
        model (`torch.nn.Module`): The model to unwrap.
4256
4257
4258
4259
4260
4261
    """
    # since there could be multiple levels of wrapping, unwrap recursively
    if hasattr(model, "module"):
        return unwrap_model(model.module)
    else:
        return model
Sylvain Gugger's avatar
Sylvain Gugger committed
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284


def expand_device_map(device_map, param_names):
    """
    Expand a device map to return the correspondance parameter name to device.
    """
    new_device_map = {}
    for module, device in device_map.items():
        new_device_map.update({p: device for p in param_names if p == module or p.startswith(f"{module}.")})
    return new_device_map


def get_disk_only_shard_files(device_map, sharded_metadata):
    """
    Returns the list of shard files containing only weights offloaded to disk.
    """
    files_content = collections.defaultdict(list)
    for weight_name, filename in sharded_metadata["weight_map"].items():
        while len(weight_name) > 0 and weight_name not in device_map:
            weight_name = ".".join(weight_name.split(".")[:-1])
        files_content[filename].append(device_map[weight_name])

    return [fname for fname, devices in files_content.items() if set(devices) == {"disk"}]