"src/lib/vscode:/vscode.git/clone" did not exist on "3b06096c522ffa91ab956cc866708823966479dd"
modeling_utils.py 238 KB
Newer Older
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team.
3
4
5
6
7
8
9
10
11
12
13
14
15
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
16
import collections
17
import copy
18
import functools
19
import gc
20
import importlib.metadata
Yih-Dar's avatar
Yih-Dar committed
21
import inspect
22
import itertools
Sylvain Gugger's avatar
Sylvain Gugger committed
23
import json
24
import os
25
import re
Sylvain Gugger's avatar
Sylvain Gugger committed
26
27
import shutil
import tempfile
28
import warnings
29
from contextlib import contextmanager
30
from dataclasses import dataclass
31
from functools import partial, wraps
32
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
33
from zipfile import is_zipfile
34
35

import torch
36
from packaging import version
Sylvain Gugger's avatar
Sylvain Gugger committed
37
from torch import Tensor, nn
38
from torch.nn import CrossEntropyLoss, Identity
39
from torch.utils.checkpoint import checkpoint
40

41
from .activations import get_activation
42
from .configuration_utils import PretrainedConfig
43
from .dynamic_module_utils import custom_object_save
44
from .generation import GenerationConfig, GenerationMixin
45
from .integrations import PeftAdapterMixin, deepspeed_config, is_deepspeed_zero3_enabled
46
47
48
49
from .pytorch_utils import (  # noqa: F401
    Conv1D,
    apply_chunking_to_forward,
    find_pruneable_heads_and_indices,
Thomas Wang's avatar
Thomas Wang committed
50
    id_tensor_storage,
51
52
53
54
    prune_conv1d_layer,
    prune_layer,
    prune_linear_layer,
)
55
from .safetensors_conversion import auto_conversion
56
from .utils import (
57
58
    ADAPTER_SAFE_WEIGHTS_NAME,
    ADAPTER_WEIGHTS_NAME,
59
    CONFIG_NAME,
Aymeric Augustin's avatar
Aymeric Augustin committed
60
    DUMMY_INPUTS,
61
    FLAX_WEIGHTS_NAME,
62
63
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
64
65
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
Sylvain Gugger's avatar
Sylvain Gugger committed
66
    WEIGHTS_INDEX_NAME,
67
    WEIGHTS_NAME,
68
    ContextManagers,
69
    ModelOutput,
Sylvain Gugger's avatar
Sylvain Gugger committed
70
    PushToHubMixin,
71
    cached_file,
72
    copy_func,
73
    download_url,
74
    extract_commit_hash,
75
    has_file,
76
    is_accelerate_available,
77
    is_auto_awq_available,
Marc Sun's avatar
Marc Sun committed
78
    is_auto_gptq_available,
79
    is_bitsandbytes_available,
80
    is_flash_attn_2_available,
81
    is_offline_mode,
82
    is_optimum_available,
83
    is_peft_available,
84
    is_remote_url,
85
    is_safetensors_available,
86
    is_torch_sdpa_available,
87
    is_torch_tpu_available,
88
    logging,
Sylvain Gugger's avatar
Sylvain Gugger committed
89
    replace_return_docstrings,
90
    strtobool,
91
)
92
from .utils.hub import convert_file_size_to_int, create_and_tag_model_card, get_checkpoint_shard_files
93
94
95
96
97
98
from .utils.import_utils import (
    ENV_VARS_TRUE_VALUES,
    is_sagemaker_mp_enabled,
    is_torch_fx_proxy,
    is_torchdynamo_compiling,
)
99
from .utils.quantization_config import AwqConfig, BitsAndBytesConfig, GPTQConfig, QuantizationMethod
100

Aymeric Augustin's avatar
Aymeric Augustin committed
101

102
103
104
XLA_USE_BF16 = os.environ.get("XLA_USE_BF16", "0").upper()
XLA_DOWNCAST_BF16 = os.environ.get("XLA_DOWNCAST_BF16", "0").upper()

105
106
if is_accelerate_available():
    from accelerate import dispatch_model, infer_auto_device_map, init_empty_weights
107
    from accelerate.hooks import add_hook_to_module
108
    from accelerate.utils import (
109
        check_tied_parameters_on_same_device,
110
        find_tied_parameters,
111
        get_balanced_memory,
Marc Sun's avatar
Marc Sun committed
112
        get_max_memory,
113
114
115
116
117
118
        load_offloaded_weights,
        offload_weight,
        save_offload_index,
        set_module_tensor_to_device,
    )

119
120
121
122
if is_safetensors_available():
    from safetensors import safe_open
    from safetensors.torch import load_file as safe_load_file
    from safetensors.torch import save_file as safe_save_file
123

Lysandre Debut's avatar
Lysandre Debut committed
124
logger = logging.get_logger(__name__)
125

126
127
128
129

_init_weights = True


130
def is_fsdp_enabled():
131
132
133
134
    return (
        torch.distributed.is_available()
        and torch.distributed.is_initialized()
        and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
135
        and strtobool(os.environ.get("FSDP_CPU_RAM_EFFICIENT_LOADING", "False")) == 1
136
    )
137
138


139
140
141
142
143
144
def is_local_dist_rank_0():
    return (
        torch.distributed.is_available()
        and torch.distributed.is_initialized()
        and int(os.environ.get("LOCAL_RANK", -1)) == 0
    )
145
146


147
148
149
150
151
152
153
154
if is_sagemaker_mp_enabled():
    import smdistributed.modelparallel.torch as smp
    from smdistributed.modelparallel import __version__ as SMP_VERSION

    IS_SAGEMAKER_MP_POST_1_10 = version.parse(SMP_VERSION) >= version.parse("1.10")
else:
    IS_SAGEMAKER_MP_POST_1_10 = False

155
156
157
if is_peft_available():
    from .utils import find_adapter_config_file

158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
TORCH_INIT_FUNCTIONS = {
    "uniform_": nn.init.uniform_,
    "normal_": nn.init.normal_,
    "trunc_normal_": nn.init.trunc_normal_,
    "constant_": nn.init.constant_,
    "xavier_uniform_": nn.init.xavier_uniform_,
    "xavier_normal_": nn.init.xavier_normal_,
    "kaiming_uniform_": nn.init.kaiming_uniform_,
    "kaiming_normal_": nn.init.kaiming_normal_,
    "uniform": nn.init.uniform,
    "normal": nn.init.normal,
    "xavier_uniform": nn.init.xavier_uniform,
    "xavier_normal": nn.init.xavier_normal,
    "kaiming_uniform": nn.init.kaiming_uniform,
    "kaiming_normal": nn.init.kaiming_normal,
}

175

176
177
178
179
180
181
182
183
@contextmanager
def no_init_weights(_enable=True):
    """
    Context manager to globally disable weight initialization to speed up loading large models.

    TODO(Patrick): Delete safety argument `_enable=True` at next major version. .
    """
    global _init_weights
184
    old_init_weights = _init_weights
185

186
187
    if _enable:
        _init_weights = False
188
189
190
191
192
193
194

        def _skip_init(*args, **kwargs):
            pass

        # # Save the original initialization functions
        for name, init_func in TORCH_INIT_FUNCTIONS.items():
            setattr(torch.nn.init, name, _skip_init)
195
196
197
    try:
        yield
    finally:
198
        _init_weights = old_init_weights
199
200
201
202
        if _enable:
            # # Restore the original initialization functions
            for name, init_func in TORCH_INIT_FUNCTIONS.items():
                setattr(torch.nn.init, name, init_func)
203
204


Lysandre Debut's avatar
Lysandre Debut committed
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
def get_parameter_device(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    try:
        return next(parameter.parameters()).device
    except StopIteration:
        # For nn.DataParallel compatibility in PyTorch 1.5

        def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
            tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
            return tuples

        gen = parameter._named_members(get_members_fn=find_tensor_attributes)
        first_tuple = next(gen)
        return first_tuple[1].device


220
221
222
223
def get_first_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    """
    Returns the first parameter dtype (can be non-floating) or asserts if none were found.
    """
Lysandre Debut's avatar
Lysandre Debut committed
224
225
226
    try:
        return next(parameter.parameters()).dtype
    except StopIteration:
Sylvain Gugger's avatar
Sylvain Gugger committed
227
        # For nn.DataParallel compatibility in PyTorch > 1.5
Lysandre Debut's avatar
Lysandre Debut committed
228
229
230
231
232
233
234
235
236
237

        def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
            tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
            return tuples

        gen = parameter._named_members(get_members_fn=find_tensor_attributes)
        first_tuple = next(gen)
        return first_tuple[1].dtype


238
239
def get_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
240
    Returns the first found floating dtype in parameters if there is one, otherwise returns the last dtype it found.
241
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
242
243
244
245
    last_dtype = None
    for t in parameter.parameters():
        last_dtype = t.dtype
        if t.is_floating_point():
246
247
248
            # Adding fix for https://github.com/pytorch/xla/issues/4152
            # Fixes issue where the model code passes a value that is out of range for XLA_USE_BF16=1
            # and XLA_DOWNCAST_BF16=1 so the conversion would cast it to -inf
249
250
251
252
253
            # NOTE: `is_torch_tpu_available()` is checked last as it induces a graph break in torch dynamo
            if XLA_USE_BF16 in ENV_VARS_TRUE_VALUES and is_torch_tpu_available():
                return torch.bfloat16
            if XLA_DOWNCAST_BF16 in ENV_VARS_TRUE_VALUES and is_torch_tpu_available():
                if t.dtype == torch.float:
254
                    return torch.bfloat16
255
256
                if t.dtype == torch.double:
                    return torch.float32
Sylvain Gugger's avatar
Sylvain Gugger committed
257
            return t.dtype
258

Sylvain Gugger's avatar
Sylvain Gugger committed
259
260
261
    if last_dtype is not None:
        # if no floating dtype was found return whatever the first dtype is
        return last_dtype
262

263
264
265
266
267
268
269
270
271
272
273
274
275
    # For nn.DataParallel compatibility in PyTorch > 1.5
    def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
        tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
        return tuples

    gen = parameter._named_members(get_members_fn=find_tensor_attributes)
    last_tuple = None
    for tuple in gen:
        last_tuple = tuple
        if tuple[1].is_floating_point():
            return tuple[1].dtype

    if last_tuple is not None:
276
277
        # fallback to the last dtype
        return last_tuple[1].dtype
278

279
280
281
282
283
284
285
    # fallback to buffer dtype
    for t in parameter.buffers():
        last_dtype = t.dtype
        if t.is_floating_point():
            return t.dtype
    return last_dtype

286
287
288
289
290
291
292
293
294
295
296
297
298
299

def get_state_dict_float_dtype(state_dict):
    """
    Returns the first found floating dtype in `state_dict` or asserts if none were found.
    """
    for t in state_dict.values():
        if t.is_floating_point():
            return t.dtype

    raise ValueError("couldn't find any floating point dtypes in state_dict")


def get_state_dict_dtype(state_dict):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
300
    Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype.
301
302
303
304
305
306
307
    """
    for t in state_dict.values():
        if t.is_floating_point():
            return t.dtype

    # if no floating dtype was found return whatever the first dtype is
    else:
Sylvain Gugger's avatar
Sylvain Gugger committed
308
        return next(state_dict.values()).dtype
309
310


Sylvain Gugger's avatar
Sylvain Gugger committed
311
312
313
314
315
316
317
318
319
320
321
322
323
def dtype_byte_size(dtype):
    """
    Returns the size (in bytes) occupied by one parameter of type `dtype`.

    Example:

    ```py
    >>> dtype_byte_size(torch.float32)
    4
    ```
    """
    if dtype == torch.bool:
        return 1 / 8
324
    bit_search = re.search(r"[^\d](\d+)$", str(dtype))
Sylvain Gugger's avatar
Sylvain Gugger committed
325
326
327
328
329
330
    if bit_search is None:
        raise ValueError(f"`dtype` is not a valid dtype: {dtype}.")
    bit_size = int(bit_search.groups()[0])
    return bit_size // 8


331
332
333
def shard_checkpoint(
    state_dict: Dict[str, torch.Tensor], max_shard_size: Union[int, str] = "10GB", weights_name: str = WEIGHTS_NAME
):
Sylvain Gugger's avatar
Sylvain Gugger committed
334
335
336
337
338
339
340
341
342
343
344
    """
    Splits a model state dictionary in sub-checkpoints so that the final size of each sub-checkpoint does not exceed a
    given size.

    The sub-checkpoints are determined by iterating through the `state_dict` in the order of its keys, so there is no
    optimization made to make each sub-checkpoint as close as possible to the maximum size passed. For example, if the
    limit is 10GB and we have weights of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB],
    [6+2+2GB] and not [6+2+2GB], [6+2GB], [6GB].

    <Tip warning={true}>

Alan Ji's avatar
Alan Ji committed
345
    If one of the model's weight is bigger than `max_shard_size`, it will end up in its own sub-checkpoint which will
Sylvain Gugger's avatar
Sylvain Gugger committed
346
347
348
349
350
351
352
353
354
    have a size greater than `max_shard_size`.

    </Tip>

    Args:
        state_dict (`Dict[str, torch.Tensor]`): The state dictionary of a model to save.
        max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`):
            The maximum size of each sub-checkpoint. If expressed as a string, needs to be digits followed by a unit
            (like `"5MB"`).
355
356
        weights_name (`str`, *optional*, defaults to `"pytorch_model.bin"`):
            The name of the model save file.
Sylvain Gugger's avatar
Sylvain Gugger committed
357
358
359
    """
    max_shard_size = convert_file_size_to_int(max_shard_size)

Thomas Wang's avatar
Thomas Wang committed
360
361
    sharded_state_dicts = [{}]
    last_block_size = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
362
    total_size = 0
Thomas Wang's avatar
Thomas Wang committed
363
    storage_id_to_block = {}
Sylvain Gugger's avatar
Sylvain Gugger committed
364
365

    for key, weight in state_dict.items():
366
367
368
369
370
371
        # when bnb serialization is used the weights in the state dict can be strings
        # check: https://github.com/huggingface/transformers/pull/24416 for more details
        if isinstance(weight, str):
            continue
        else:
            storage_id = id_tensor_storage(weight)
Thomas Wang's avatar
Thomas Wang committed
372
373
374
375
376
377
378

        # If a `weight` shares the same underlying storage as another tensor, we put `weight` in the same `block`
        if storage_id in storage_id_to_block:
            block_id = storage_id_to_block[storage_id]
            sharded_state_dicts[block_id][key] = weight
            continue

Sylvain Gugger's avatar
Sylvain Gugger committed
379
380
        weight_size = weight.numel() * dtype_byte_size(weight.dtype)

Sylvain Gugger's avatar
Sylvain Gugger committed
381
382
383
        # If this weight is going to tip up over the maximal size, we split, but only if we have put at least one
        # weight in the current shard.
        if last_block_size + weight_size > max_shard_size and len(sharded_state_dicts[-1]) > 0:
Thomas Wang's avatar
Thomas Wang committed
384
385
            sharded_state_dicts.append({})
            last_block_size = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
386

Thomas Wang's avatar
Thomas Wang committed
387
388
        sharded_state_dicts[-1][key] = weight
        last_block_size += weight_size
Sylvain Gugger's avatar
Sylvain Gugger committed
389
        total_size += weight_size
Thomas Wang's avatar
Thomas Wang committed
390
        storage_id_to_block[storage_id] = len(sharded_state_dicts) - 1
Sylvain Gugger's avatar
Sylvain Gugger committed
391
392
393

    # If we only have one shard, we return it
    if len(sharded_state_dicts) == 1:
394
        return {weights_name: sharded_state_dicts[0]}, None
Sylvain Gugger's avatar
Sylvain Gugger committed
395
396
397
398
399

    # Otherwise, let's build the index
    weight_map = {}
    shards = {}
    for idx, shard in enumerate(sharded_state_dicts):
400
401
402
403
        shard_file = weights_name.replace(".bin", f"-{idx+1:05d}-of-{len(sharded_state_dicts):05d}.bin")
        shard_file = shard_file.replace(
            ".safetensors", f"-{idx + 1:05d}-of-{len(sharded_state_dicts):05d}.safetensors"
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
404
405
406
407
408
409
410
411
412
413
        shards[shard_file] = shard
        for key in shard.keys():
            weight_map[key] = shard_file

    # Add the metadata
    metadata = {"total_size": total_size}
    index = {"metadata": metadata, "weight_map": weight_map}
    return shards, index


414
def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True):
415
416
417
418
419
420
421
422
423
424
425
426
427
    """
    This is the same as
    [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict)
    but for a sharded checkpoint.

    This load is performed efficiently: each checkpoint shard is loaded one by one in RAM and deleted after being
    loaded in the model.

    Args:
        model (`torch.nn.Module`): The model in which to load the checkpoint.
        folder (`str` or `os.PathLike`): A path to a folder containing the sharded checkpoint.
        strict (`bool`, *optional`, defaults to `True`):
            Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint.
428
429
430
        prefer_safe (`bool`, *optional*, defaults to `False`)
            If both safetensors and PyTorch save files are present in checkpoint and `prefer_safe` is True, the
            safetensors files will be loaded. Otherwise, PyTorch files are always loaded when possible.
431
432
433
434
435
436
437
438

    Returns:
        `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields
            - `missing_keys` is a list of str containing the missing keys
            - `unexpected_keys` is a list of str containing the unexpected keys
    """
    # Load the index
    index_file = os.path.join(folder, WEIGHTS_INDEX_NAME)
439
    safe_index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME)
440

441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
    index_present = os.path.isfile(index_file)
    safe_index_present = os.path.isfile(safe_index_file)

    if not index_present and not (safe_index_present and is_safetensors_available()):
        filenames = (
            (WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_INDEX_NAME) if is_safetensors_available() else (WEIGHTS_INDEX_NAME,)
        )
        raise ValueError(f"Can't find a checkpoint index ({' or '.join(filenames)}) in {folder}.")

    load_safe = False
    if safe_index_present:
        if prefer_safe:
            if is_safetensors_available():
                load_safe = True  # load safe due to preference
            else:
                logger.warning(
                    f"Cannot load sharded checkpoint at {folder} safely since safetensors is not installed!"
                )
        elif not index_present:
            load_safe = True  # load safe since we have no other choice

    load_index = safe_index_file if load_safe else index_file

    with open(load_index, "r", encoding="utf-8") as f:
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
        index = json.load(f)

    shard_files = list(set(index["weight_map"].values()))

    # If strict=True, error before loading any of the state dicts.
    loaded_keys = index["weight_map"].keys()
    model_keys = model.state_dict().keys()
    missing_keys = [key for key in model_keys if key not in loaded_keys]
    unexpected_keys = [key for key in loaded_keys if key not in model_keys]
    if strict and (len(missing_keys) > 0 or len(unexpected_keys) > 0):
        error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}"
        if len(missing_keys) > 0:
            str_missing_keys = ",".join([f'"{k}"' for k in missing_keys])
            error_message += f"\nMissing key(s): {str_missing_keys}."
        if len(unexpected_keys) > 0:
            str_unexpected_keys = ",".join([f'"{k}"' for k in unexpected_keys])
            error_message += f"\nMissing key(s): {str_unexpected_keys}."
        raise RuntimeError(error_message)

484
    loader = safe_load_file if load_safe else partial(torch.load, map_location="cpu", weights_only=True)
485

486
    for shard_file in shard_files:
487
        state_dict = loader(os.path.join(folder, shard_file))
488
489
        model.load_state_dict(state_dict, strict=False)

490
        # Make sure memory is freed before we load the next state dict.
491
492
493
494
495
496
497
        del state_dict
        gc.collect()

    # Return the same thing as PyTorch load_state_dict function.
    return torch.nn.modules.module._IncompatibleKeys(missing_keys, unexpected_keys)


Sylvain Gugger's avatar
Sylvain Gugger committed
498
499
500
501
def load_state_dict(checkpoint_file: Union[str, os.PathLike]):
    """
    Reads a PyTorch checkpoint file, returning properly formatted errors if they arise.
    """
502
503
504
505
506
507
508
509
510
511
    if checkpoint_file.endswith(".safetensors") and is_safetensors_available():
        # Check format of the archive
        with safe_open(checkpoint_file, framework="pt") as f:
            metadata = f.metadata()
        if metadata.get("format") not in ["pt", "tf", "flax"]:
            raise OSError(
                f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure "
                "you save your model with the `save_pretrained` method."
            )
        return safe_load_file(checkpoint_file)
Sylvain Gugger's avatar
Sylvain Gugger committed
512
    try:
513
        if (
514
515
            is_deepspeed_zero3_enabled() and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0
        ) or (is_fsdp_enabled() and not is_local_dist_rank_0()):
516
517
518
            map_location = "meta"
        else:
            map_location = "cpu"
519
520
521
522
523
524
525
526
527
528
        extra_args = {}
        # mmap can only be used with files serialized with zipfile-based format.
        if (
            isinstance(checkpoint_file, str)
            and map_location != "meta"
            and version.parse(torch.__version__) >= version.parse("2.1.0")
            and is_zipfile(checkpoint_file)
        ):
            extra_args = {"mmap": True}
        return torch.load(checkpoint_file, map_location=map_location, weights_only=True, **extra_args)
Sylvain Gugger's avatar
Sylvain Gugger committed
529
530
531
    except Exception as e:
        try:
            with open(checkpoint_file) as f:
532
                if f.read(7) == "version":
Sylvain Gugger's avatar
Sylvain Gugger committed
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
                    raise OSError(
                        "You seem to have cloned a repository without having git-lfs installed. Please install "
                        "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder "
                        "you cloned."
                    )
                else:
                    raise ValueError(
                        f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained "
                        "model. Make sure you have saved the model properly."
                    ) from e
        except (UnicodeDecodeError, ValueError):
            raise OSError(
                f"Unable to load weights from pytorch checkpoint file for '{checkpoint_file}' "
                f"at '{checkpoint_file}'. "
                "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True."
            )


551
552
553
554
555
def set_initialized_submodules(model, state_dict_keys):
    """
    Sets the `_is_hf_initialized` flag in all submodules of a given model when all its weights are in the loaded state
    dict.
    """
556
    not_initialized_submodules = {}
557
    for module_name, module in model.named_modules():
558
559
        loaded_keys = {k.replace(f"{module_name}.", "") for k in state_dict_keys if k.startswith(f"{module_name}.")}
        if loaded_keys.issuperset(module.state_dict()):
560
            module._is_hf_initialized = True
561
562
563
        else:
            not_initialized_submodules[module_name] = module
    return not_initialized_submodules
564
565


Sylvain Gugger's avatar
Sylvain Gugger committed
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
def _load_state_dict_into_model(model_to_load, state_dict, start_prefix):
    # Convert old format to new format if needed from a PyTorch state_dict
    old_keys = []
    new_keys = []
    for key in state_dict.keys():
        new_key = None
        if "gamma" in key:
            new_key = key.replace("gamma", "weight")
        if "beta" in key:
            new_key = key.replace("beta", "bias")
        if new_key:
            old_keys.append(key)
            new_keys.append(new_key)
    for old_key, new_key in zip(old_keys, new_keys):
        state_dict[new_key] = state_dict.pop(old_key)

    # copy state_dict so _load_from_state_dict can modify it
    metadata = getattr(state_dict, "_metadata", None)
    state_dict = state_dict.copy()
    if metadata is not None:
        state_dict._metadata = metadata

    error_msgs = []

    # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants
    # so we need to apply the function recursively.
592
    def load(module: nn.Module, state_dict, prefix=""):
Sylvain Gugger's avatar
Sylvain Gugger committed
593
594
        local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
        args = (state_dict, prefix, local_metadata, True, [], [], error_msgs)
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
        # Parameters of module and children will start with prefix. We can exit early if there are none in this
        # state_dict
        if len([key for key in state_dict if key.startswith(prefix)]) > 0:
            if is_deepspeed_zero3_enabled():
                import deepspeed

                # In sharded models, each shard has only part of the full state_dict, so only gather
                # parameters that are in the current state_dict.
                named_parameters = dict(module.named_parameters(prefix=prefix[:-1], recurse=False))
                params_to_gather = [named_parameters[k] for k in state_dict.keys() if k in named_parameters]
                if len(params_to_gather) > 0:
                    # because zero3 puts placeholders in model params, this context
                    # manager gathers (unpartitions) the params of the current layer, then loads from
                    # the state dict and then re-partitions them again
                    with deepspeed.zero.GatheredParameters(params_to_gather, modifier_rank=0):
                        if torch.distributed.get_rank() == 0:
                            module._load_from_state_dict(*args)
            else:
                module._load_from_state_dict(*args)
Sylvain Gugger's avatar
Sylvain Gugger committed
614
615
616

        for name, child in module._modules.items():
            if child is not None:
617
                load(child, state_dict, prefix + name + ".")
Sylvain Gugger's avatar
Sylvain Gugger committed
618

619
620
621
622
    load(model_to_load, state_dict, prefix=start_prefix)
    # Delete `state_dict` so it could be collected by GC earlier. Note that `state_dict` is a copy of the argument, so
    # it's safe to delete it.
    del state_dict
Sylvain Gugger's avatar
Sylvain Gugger committed
623
624
625
626

    return error_msgs


627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
def find_submodule_and_param_name(model, long_key, start_prefix):
    """
    A helper util to find the last sub-module and the param/buffer name. If `start_prefix` is supplied it'll be removed
    from the start of the key
    """

    if len(start_prefix) > 0 and long_key.startswith(start_prefix):
        long_key = ".".join(long_key.split(".")[1:])

    split_key = long_key.split(".")
    submodule = model
    while len(split_key) > 1:
        if hasattr(submodule, split_key[0]):
            submodule = getattr(submodule, split_key[0])
            del split_key[0]
        else:
            submodule = None
            break
    if submodule == model:
        submodule = None
    return submodule, split_key[0]


def _move_model_to_meta(model, loaded_state_dict_keys, start_prefix):
    """
    Moves `loaded_state_dict_keys` in model to meta device which frees up the memory taken by those params.

    `start_prefix` is used for models which insert their name into model keys, e.g. `bert` in
    `bert.pooler.dense.weight`

    """

    # dematerialize param storage for keys that are going to be replaced by state_dict, by
    # putting those on the meta device
    for k in loaded_state_dict_keys:
        submodule, param_name = find_submodule_and_param_name(model, k, start_prefix)
        if submodule is not None:
            # selectively switch to the meta device only those params/buffers that will
            # be next replaced from state_dict. This a complex way to do p.to_("meta")
            # since we have no in-place to_ for tensors.
            new_val = getattr(submodule, param_name)
            if isinstance(new_val, torch.nn.Parameter):
                # isinstance returns False for Params on meta device, so switch after the check
                new_val = torch.nn.Parameter(new_val.to("meta"))
            else:
                new_val = new_val.to("meta")
            setattr(submodule, param_name, new_val)


676
677
678
679
680
681
682
683
684
685
686
687
def _load_state_dict_into_meta_model(
    model,
    state_dict,
    loaded_state_dict_keys,  # left for now but could be removed, see below
    start_prefix,
    expected_keys,
    device_map=None,
    offload_folder=None,
    offload_index=None,
    state_dict_folder=None,
    state_dict_index=None,
    dtype=None,
688
    is_quantized=False,
Sylvain Gugger's avatar
Sylvain Gugger committed
689
    is_safetensors=False,
690
    keep_in_fp32_modules=None,
691
    unexpected_keys=None,  # passing `unexpected` for cleanup from quantization items
692
):
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
    """
    This is somewhat similar to `_load_state_dict_into_model`, but deals with a model that has some or all of its
    params on a `meta` device. It replaces the model params with the data from the `state_dict`, while moving the
    params back to the normal device, but only for `loaded_state_dict_keys`.

    `start_prefix` is used for models which insert their name into model keys, e.g. `bert` in
    `bert.pooler.dense.weight`

    """

    # XXX: remaining features to implement to be fully compatible with _load_state_dict_into_model
    # - deepspeed zero 3 support
    # - need to copy metadata if any - see _load_state_dict_into_model
    # - handling error_msgs - mimicking the error handling in module._load_from_state_dict()
    # - Is there a situation where some keys aren't in `loaded_state_dict_keys` and in which case
    #   they won't get loaded.

710
    if is_quantized:
711
        from .integrations import set_module_quantized_tensor_to_device
712

713
714
    error_msgs = []

715
716
717
718
719
720
721
722
723
724
725
726
727
    old_keys = []
    new_keys = []
    for key in state_dict.keys():
        new_key = None
        if "gamma" in key:
            new_key = key.replace("gamma", "weight")
        if "beta" in key:
            new_key = key.replace("beta", "bias")
        if new_key:
            old_keys.append(key)
            new_keys.append(new_key)
    for old_key, new_key in zip(old_keys, new_keys):
        state_dict[new_key] = state_dict.pop(old_key)
728

729
730
731
732
733
734
735
736
737
    for param_name, param in state_dict.items():
        # First part of the test is always true as load_state_dict_keys always contains state_dict keys.
        if param_name not in loaded_state_dict_keys or param_name not in expected_keys:
            continue

        if param_name.startswith(start_prefix):
            param_name = param_name[len(start_prefix) :]

        module_name = param_name
738
        set_module_kwargs = {}
739

740
        # We convert floating dtypes to the `dtype` passed. We want to keep the buffers/params
741
742
        # in int/uint/bool and not cast them.
        if dtype is not None and torch.is_floating_point(param):
743
744
            if (
                keep_in_fp32_modules is not None
745
746
747
                and any(
                    module_to_keep_in_fp32 in param_name.split(".") for module_to_keep_in_fp32 in keep_in_fp32_modules
                )
748
749
750
                and dtype == torch.float16
            ):
                param = param.to(torch.float32)
751
752
753
754
755

                # For backward compatibility with older versions of `accelerate`
                # TODO: @sgugger replace this check with version check at the next `accelerate` release
                if "dtype" in list(inspect.signature(set_module_tensor_to_device).parameters):
                    set_module_kwargs["dtype"] = torch.float32
756
757
            else:
                param = param.to(dtype)
758
759
760
761
762
763
764
765
766
767
768
769

        # For compatibility with PyTorch load_state_dict which converts state dict dtype to existing dtype in model
        if dtype is None:
            old_param = model
            splits = param_name.split(".")
            for split in splits:
                old_param = getattr(old_param, split)
                if old_param is None:
                    break

            if old_param is not None:
                param = param.to(old_param.dtype)
770

771
772
        set_module_kwargs["value"] = param

773
774
775
776
777
778
779
780
781
782
783
        if device_map is None:
            param_device = "cpu"
        else:
            # find next higher level module that is defined in device_map:
            # bert.lm_head.weight -> bert.lm_head -> bert -> ''
            while len(module_name) > 0 and module_name not in device_map:
                module_name = ".".join(module_name.split(".")[:-1])
            if module_name == "" and "" not in device_map:
                # TODO: group all errors and raise at the end.
                raise ValueError(f"{param_name} doesn't have any device set.")
            param_device = device_map[module_name]
784

785
        if param_device == "disk":
Sylvain Gugger's avatar
Sylvain Gugger committed
786
787
            if not is_safetensors:
                offload_index = offload_weight(param, param_name, offload_folder, offload_index)
788
789
        elif param_device == "cpu" and state_dict_index is not None:
            state_dict_index = offload_weight(param, param_name, state_dict_folder, state_dict_index)
790
        elif not is_quantized:
791
792
            # For backward compatibility with older versions of `accelerate`
            set_module_tensor_to_device(model, param_name, param_device, **set_module_kwargs)
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
        elif param.dtype in (torch.int8, torch.uint8) and is_quantized:
            # handling newly quantized weights and loaded quantized weights
            # edit the param.dtype restrictions and is_quantized condition when adding new quant methods
            quantized_stats = {}

            if (param_name + ".quant_state.bitsandbytes__fp4" in state_dict) or (
                param_name + ".quant_state.bitsandbytes__nf4" in state_dict
            ):
                # 4bit loading. Collecting components for restoring quantized weight
                # This can be expanded to make a universal call for any quantized weight loading
                for k, v in state_dict.items():
                    if param_name + "." in k:
                        quantized_stats[k] = v
                        unexpected_keys.remove(k)

                set_module_quantized_tensor_to_device(
                    model, param_name, param_device, value=param, quantized_stats=quantized_stats
                )
811

812
813
814
815
816
            elif param.dtype == torch.int8 and param_name.replace("weight", "SCB") in state_dict.keys():
                # 8bit loading. Could be combined with the above 4bit call.
                # condition looks unreliable
                fp16_statistics_key = param_name.replace("weight", "SCB")
                unexpected_keys.remove(fp16_statistics_key)
817
                set_module_quantized_tensor_to_device(
818
819
820
821
822
                    model,
                    param_name,
                    param_device,
                    value=param,
                    quantized_stats={"SCB": state_dict[fp16_statistics_key]},
823
                )
824
825
826
        else:
            # loading not quantized params in quantized model
            set_module_quantized_tensor_to_device(model, param_name, param_device, value=param)
827
828

    return error_msgs, offload_index, state_dict_index
829
830


831
832
833
834
835
836
837
838
839
def _add_variant(weights_name: str, variant: Optional[str] = None) -> str:
    if variant is not None:
        splits = weights_name.split(".")
        splits = splits[:-1] + [variant] + splits[-1:]
        weights_name = ".".join(splits)

    return weights_name


840
class ModuleUtilsMixin:
Julien Chaumond's avatar
Julien Chaumond committed
841
    """
842
    A few utilities for `torch.nn.Modules`, to be used as a mixin.
Julien Chaumond's avatar
Julien Chaumond committed
843
844
    """

845
846
847
848
    @staticmethod
    def _hook_rss_memory_pre_forward(module, *args, **kwargs):
        try:
            import psutil
Sylvain Gugger's avatar
Sylvain Gugger committed
849
        except ImportError:
850
851
852
853
854
855
856
857
858
859
860
            raise ImportError("You need to install psutil (pip install psutil) to use memory tracing.")

        process = psutil.Process(os.getpid())
        mem = process.memory_info()
        module.mem_rss_pre_forward = mem.rss
        return None

    @staticmethod
    def _hook_rss_memory_post_forward(module, *args, **kwargs):
        try:
            import psutil
Sylvain Gugger's avatar
Sylvain Gugger committed
861
        except ImportError:
862
863
864
865
866
867
868
869
870
871
            raise ImportError("You need to install psutil (pip install psutil) to use memory tracing.")

        process = psutil.Process(os.getpid())
        mem = process.memory_info()
        module.mem_rss_post_forward = mem.rss
        mem_rss_diff = module.mem_rss_post_forward - module.mem_rss_pre_forward
        module.mem_rss_diff = mem_rss_diff + (module.mem_rss_diff if hasattr(module, "mem_rss_diff") else 0)
        return None

    def add_memory_hooks(self):
Sylvain Gugger's avatar
Sylvain Gugger committed
872
873
874
        """
        Add a memory hook before and after each sub-module forward pass to record increase in memory consumption.

Sylvain Gugger's avatar
Sylvain Gugger committed
875
876
        Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to zero
        with `model.reset_memory_hooks_state()`.
877
878
879
880
881
882
883
        """
        for module in self.modules():
            module.register_forward_pre_hook(self._hook_rss_memory_pre_forward)
            module.register_forward_hook(self._hook_rss_memory_post_forward)
        self.reset_memory_hooks_state()

    def reset_memory_hooks_state(self):
Sylvain Gugger's avatar
Sylvain Gugger committed
884
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
885
        Reset the `mem_rss_diff` attribute of each module (see [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]).
Sylvain Gugger's avatar
Sylvain Gugger committed
886
        """
887
888
889
890
891
        for module in self.modules():
            module.mem_rss_diff = 0
            module.mem_rss_post_forward = 0
            module.mem_rss_pre_forward = 0

892
    @property
Sylvain Gugger's avatar
Sylvain Gugger committed
893
    def device(self) -> torch.device:
894
        """
895
        `torch.device`: The device on which the module is (assuming that all the module parameters are on the same
896
        device).
897
        """
Lysandre Debut's avatar
Lysandre Debut committed
898
        return get_parameter_device(self)
899

900
    @property
901
    def dtype(self) -> torch.dtype:
902
        """
903
        `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
904
        """
Lysandre Debut's avatar
Lysandre Debut committed
905
        return get_parameter_dtype(self)
906
907

    def invert_attention_mask(self, encoder_attention_mask: Tensor) -> Tensor:
Sylvain Gugger's avatar
Sylvain Gugger committed
908
909
910
911
        """
        Invert an attention mask (e.g., switches 0. and 1.).

        Args:
912
            encoder_attention_mask (`torch.Tensor`): An attention mask.
Sylvain Gugger's avatar
Sylvain Gugger committed
913
914

        Returns:
915
            `torch.Tensor`: The inverted attention mask.
Sylvain Gugger's avatar
Sylvain Gugger committed
916
        """
917
918
919
920
921
922
923
924
925
926
        if encoder_attention_mask.dim() == 3:
            encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
        if encoder_attention_mask.dim() == 2:
            encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
        # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
        # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
        # /transformer/transformer_layers.py#L270
        # encoder_extended_attention_mask = (encoder_extended_attention_mask ==
        # encoder_extended_attention_mask.transpose(-1, -2))
        encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
Yih-Dar's avatar
Yih-Dar committed
927
        encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * torch.finfo(self.dtype).min
928

929
930
        return encoder_extended_attention_mask

931
    @staticmethod
932
933
934
935
936
937
938
    def create_extended_attention_mask_for_decoder(input_shape, attention_mask, device=None):
        if device is not None:
            warnings.warn(
                "The `device` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
            )
        else:
            device = attention_mask.device
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
        batch_size, seq_length = input_shape
        seq_ids = torch.arange(seq_length, device=device)
        causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
        # in case past_key_values are used we need to add a prefix ones mask to the causal mask
        # causal and attention masks must have same type with pytorch version < 1.3
        causal_mask = causal_mask.to(attention_mask.dtype)

        if causal_mask.shape[1] < attention_mask.shape[1]:
            prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
            causal_mask = torch.cat(
                [
                    torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
                    causal_mask,
                ],
                axis=-1,
            )

        extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
        return extended_attention_mask

959
    def get_extended_attention_mask(
960
        self, attention_mask: Tensor, input_shape: Tuple[int], device: torch.device = None, dtype: torch.float = None
961
    ) -> Tensor:
Sylvain Gugger's avatar
Sylvain Gugger committed
962
963
        """
        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
964
965

        Arguments:
966
            attention_mask (`torch.Tensor`):
Sylvain Gugger's avatar
Sylvain Gugger committed
967
                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
968
            input_shape (`Tuple[int]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
969
                The shape of the input to the model.
970
971

        Returns:
972
            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
973
        """
Yih-Dar's avatar
Yih-Dar committed
974
975
976
        if dtype is None:
            dtype = self.dtype

977
978
979
980
981
982
        if not (attention_mask.dim() == 2 and self.config.is_decoder):
            # show warning only if it won't be shown in `create_extended_attention_mask_for_decoder`
            if device is not None:
                warnings.warn(
                    "The `device` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
                )
983
984
985
986
987
988
989
990
991
        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        if attention_mask.dim() == 3:
            extended_attention_mask = attention_mask[:, None, :, :]
        elif attention_mask.dim() == 2:
            # Provided a padding mask of dimensions [batch_size, seq_length]
            # - if the model is a decoder, apply a causal mask in addition to the padding mask
            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
            if self.config.is_decoder:
992
                extended_attention_mask = ModuleUtilsMixin.create_extended_attention_mask_for_decoder(
993
994
                    input_shape, attention_mask, device
                )
995
996
997
998
            else:
                extended_attention_mask = attention_mask[:, None, None, :]
        else:
            raise ValueError(
999
                f"Wrong shape for input_ids (shape {input_shape}) or attention_mask (shape {attention_mask.shape})"
1000
1001
1002
1003
            )

        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
        # masked positions, this operation will create a tensor which is 0.0 for
1004
        # positions we want to attend and the dtype's smallest value for masked positions.
1005
1006
        # Since we are adding it to the raw scores before the softmax, this is
        # effectively the same as removing these entirely.
Yih-Dar's avatar
Yih-Dar committed
1007
1008
        extended_attention_mask = extended_attention_mask.to(dtype=dtype)  # fp16 compatibility
        extended_attention_mask = (1.0 - extended_attention_mask) * torch.finfo(dtype).min
1009
1010
        return extended_attention_mask

Sylvain Gugger's avatar
Sylvain Gugger committed
1011
1012
1013
    def get_head_mask(
        self, head_mask: Optional[Tensor], num_hidden_layers: int, is_attention_chunked: bool = False
    ) -> Tensor:
1014
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
1015
1016
1017
        Prepare the head mask if needed.

        Args:
1018
            head_mask (`torch.Tensor` with shape `[num_heads]` or `[num_hidden_layers x num_heads]`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
1019
                The mask indicating if we should keep the heads or not (1.0 for keep, 0.0 for discard).
1020
            num_hidden_layers (`int`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1021
                The number of hidden layers in the model.
1022
            is_attention_chunked (`bool`, *optional*, defaults to `False`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1023
1024
                Whether or not the attentions scores are computed by chunks or not.

1025
        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
1026
1027
            `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or list with
            `[None]` for each layer.
1028
1029
1030
        """
        if head_mask is not None:
            head_mask = self._convert_head_mask_to_5d(head_mask, num_hidden_layers)
Patrick von Platen's avatar
Patrick von Platen committed
1031
1032
            if is_attention_chunked is True:
                head_mask = head_mask.unsqueeze(-1)
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
        else:
            head_mask = [None] * num_hidden_layers

        return head_mask

    def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers):
        """-> [num_hidden_layers x batch x num_heads x seq_length x seq_length]"""
        if head_mask.dim() == 1:
            head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
            head_mask = head_mask.expand(num_hidden_layers, -1, -1, -1, -1)
        elif head_mask.dim() == 2:
            head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
        assert head_mask.dim() == 5, f"head_mask.dim != 5, instead {head_mask.dim()}"
1046
        head_mask = head_mask.to(dtype=self.dtype)  # switch to float if need + fp16 compatibility
1047
1048
        return head_mask

1049
1050
1051
1052
1053
    def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int:
        """
        Get number of (optionally, trainable or non-embeddings) parameters in the module.

        Args:
1054
            only_trainable (`bool`, *optional*, defaults to `False`):
1055
1056
                Whether or not to return only the number of trainable parameters

1057
            exclude_embeddings (`bool`, *optional*, defaults to `False`):
1058
1059
1060
                Whether or not to return only the number of non-embeddings parameters

        Returns:
1061
            `int`: The number of parameters.
1062
1063
        """

1064
1065
1066
1067
        if exclude_embeddings:
            embedding_param_names = [
                f"{name}.weight" for name, module_type in self.named_modules() if isinstance(module_type, nn.Embedding)
            ]
1068
            total_parameters = [
1069
1070
1071
                parameter for name, parameter in self.named_parameters() if name not in embedding_param_names
            ]
        else:
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
            total_parameters = list(self.parameters())

        total_numel = []
        is_loaded_in_4bit = getattr(self, "is_loaded_in_4bit", False)
        if is_loaded_in_4bit:
            if is_bitsandbytes_available():
                import bitsandbytes as bnb
            else:
                raise ValueError(
                    "bitsandbytes is not installed but it seems that the model has been loaded in 4bit precision, something went wrong"
1082
                    " make sure to install bitsandbytes with `pip install bitsandbytes`. You also need a GPU. "
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
                )

        for param in total_parameters:
            if param.requires_grad or not only_trainable:
                # For 4bit models, we need to multiply the number of parameters by 2 as half of the parameters are
                # used for the 4bit quantization (uint8 tensors are stored)
                if is_loaded_in_4bit and isinstance(param, bnb.nn.Params4bit):
                    total_numel.append(param.numel() * 2)
                else:
                    total_numel.append(param.numel())

        return sum(total_numel)
1095
1096
1097
1098
1099
1100

    def estimate_tokens(self, input_dict: Dict[str, Union[torch.Tensor, Any]]) -> int:
        """
        Helper function to estimate the total number of tokens from the model inputs.

        Args:
1101
            inputs (`dict`): The model inputs.
1102
1103

        Returns:
1104
            `int`: The total number of tokens.
1105
        """
1106
1107
        if not hasattr(self, "warnings_issued"):
            self.warnings_issued = {}
1108
1109
        if self.main_input_name in input_dict:
            return input_dict[self.main_input_name].numel()
1110
        elif "estimate_tokens" not in self.warnings_issued:
1111
            logger.warning(
1112
1113
                "Could not estimate the number of tokens of the input, floating-point operations will not be computed"
            )
1114
1115
            self.warnings_issued["estimate_tokens"] = True
        return 0
1116
1117
1118
1119
1120
1121
1122

    def floating_point_ops(
        self, input_dict: Dict[str, Union[torch.Tensor, Any]], exclude_embeddings: bool = True
    ) -> int:
        """
        Get number of (optionally, non-embeddings) floating-point operations for the forward and backward passes of a
        batch with this transformer model. Default approximation neglects the quadratic dependency on the number of
Sylvain Gugger's avatar
Sylvain Gugger committed
1123
1124
        tokens (valid if `12 * d_model << sequence_length`) as laid out in [this
        paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter
Sylvain Gugger's avatar
Sylvain Gugger committed
1125
        re-use e.g. Albert or Universal Transformers, or if doing long-range modeling with very high sequence lengths.
1126
1127

        Args:
1128
            batch_size (`int`):
1129
1130
                The batch size for the forward pass.

1131
            sequence_length (`int`):
1132
1133
                The number of tokens in each line of the batch.

1134
            exclude_embeddings (`bool`, *optional*, defaults to `True`):
1135
1136
1137
                Whether or not to count embedding and softmax operations.

        Returns:
1138
            `int`: The number of floating-point operations.
1139
1140
1141
1142
        """

        return 6 * self.estimate_tokens(input_dict) * self.num_parameters(exclude_embeddings=exclude_embeddings)

Julien Chaumond's avatar
Julien Chaumond committed
1143

1144
class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMixin, PeftAdapterMixin):
1145
1146
    r"""
    Base class for all models.
1147

Sylvain Gugger's avatar
Sylvain Gugger committed
1148
1149
    [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading,
    downloading and saving models as well as a few methods common to all models to:
1150

1151
1152
        - resize the input embeddings,
        - prune heads in the self-attention heads.
1153

1154
    Class attributes (overridden by derived classes):
Sylvain Gugger's avatar
Sylvain Gugger committed
1155

Sylvain Gugger's avatar
Sylvain Gugger committed
1156
1157
1158
1159
        - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class
          for this model architecture.
        - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch model,
          taking as arguments:
1160

Sylvain Gugger's avatar
Sylvain Gugger committed
1161
1162
            - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the TensorFlow checkpoint.
            - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to the model.
1163
            - **path** (`str`) -- A path to the TensorFlow checkpoint.
1164

Sylvain Gugger's avatar
Sylvain Gugger committed
1165
1166
        - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived
          classes of the same architecture adding modules on top of the base model.
1167
        - **is_parallelizable** (`bool`) -- A flag indicating whether this model supports model parallelization.
Sylvain Gugger's avatar
Sylvain Gugger committed
1168
1169
        - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP
          models, `pixel_values` for vision models and `input_values` for speech models).
1170
    """
1171

1172
    config_class = None
1173
    base_model_prefix = ""
1174
    main_input_name = "input_ids"
1175
1176
    model_tags = None

1177
    _auto_class = None
1178
    _no_split_modules = None
1179
    _skip_keys_device_placement = None
1180
    _keep_in_fp32_modules = None
1181

1182
1183
    # a list of `re` patterns of `state_dict` keys that should be removed from the list of missing
    # keys we find (keys inside the model but not in the checkpoint) and avoid unnecessary warnings.
1184
    _keys_to_ignore_on_load_missing = None
1185
1186
1187
    # a list of `re` patterns of `state_dict` keys that should be removed from the list of
    # unexpected keys we find (keys inside the checkpoint but not the model) and avoid unnecessary
    # warnings.
1188
    _keys_to_ignore_on_load_unexpected = None
1189
1190
    # a list of `state_dict` keys to ignore when saving the model (useful for keys that aren't
    # trained, but which are either deterministic or tied variables)
1191
    _keys_to_ignore_on_save = None
Sylvain Gugger's avatar
Sylvain Gugger committed
1192
1193
    # a list of `state_dict` keys that are potentially tied to another key in the state_dict.
    _tied_weights_keys = None
1194

1195
    is_parallelizable = False
1196
    supports_gradient_checkpointing = False
1197

1198
1199
1200
    # Flash Attention 2 support
    _supports_flash_attn_2 = False

1201
1202
1203
    # SDPA support
    _supports_sdpa = False

1204
1205
1206
    # Has support for a `Cache` instance as `past_key_values`
    _supports_cache_class = False

1207
    @property
1208
    def dummy_inputs(self) -> Dict[str, torch.Tensor]:
1209
        """
1210
        `Dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network.
1211
        """
1212
        return {"input_ids": torch.tensor(DUMMY_INPUTS)}
1213

1214
1215
1216
1217
1218
1219
1220
    @property
    def framework(self) -> str:
        """
        :str: Identifies that this is a PyTorch model.
        """
        return "pt"

1221
    def __init__(self, config: PretrainedConfig, *inputs, **kwargs):
Julien Chaumond's avatar
Julien Chaumond committed
1222
        super().__init__()
1223
1224
        if not isinstance(config, PretrainedConfig):
            raise ValueError(
1225
1226
1227
                f"Parameter config in `{self.__class__.__name__}(config)` should be an instance of class "
                "`PretrainedConfig`. To create a model from a pretrained model use "
                f"`model = {self.__class__.__name__}.from_pretrained(PRETRAINED_MODEL_NAME)`"
1228
            )
1229
        # Save config and origin of the pretrained weights if given in model
1230
1231
1232
        config = self._autoset_attn_implementation(
            config, torch_dtype=torch.get_default_dtype(), check_device_map=False
        )
1233
        self.config = config
1234

1235
        self.name_or_path = config.name_or_path
1236
        self.warnings_issued = {}
1237
        self.generation_config = GenerationConfig.from_model_config(config) if self.can_generate() else None
1238
1239
1240
1241
        # Overwrite the class attribute to make it an instance attribute, so models like
        # `InstructBlipForConditionalGeneration` can dynamically update it without modifying the class attribute
        # when a different component (e.g. language_model) is used.
        self._keep_in_fp32_modules = copy.copy(self.__class__._keep_in_fp32_modules)
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255

    def post_init(self):
        """
        A method executed at the end of each Transformer model initialization, to execute code that needs the model's
        modules properly initialized (such as weight initialization).
        """
        self.init_weights()
        self._backward_compatibility_gradient_checkpointing()

    def _backward_compatibility_gradient_checkpointing(self):
        if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False):
            self.gradient_checkpointing_enable()
            # Remove the attribute now that is has been consumed, so it's no saved in the config.
            delattr(self.config, "gradient_checkpointing")
1256

1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
    def add_model_tags(self, tags: Union[List[str], str]) -> None:
        r"""
        Add custom tags into the model that gets pushed to the Hugging Face Hub. Will
        not overwrite existing tags in the model.

        Args:
            tags (`Union[List[str], str]`):
                The desired tags to inject in the model

        Examples:

        ```python
        from transformers import AutoModel

        model = AutoModel.from_pretrained("bert-base-cased")

        model.add_model_tags(["custom", "custom-bert"])

        # Push the model to your namespace with the name "my-custom-bert".
        model.push_to_hub("my-custom-bert")
        ```
        """
        if isinstance(tags, str):
            tags = [tags]

        if self.model_tags is None:
            self.model_tags = []

        for tag in tags:
            if tag not in self.model_tags:
                self.model_tags.append(tag)

1289
1290
1291
1292
1293
1294
    @classmethod
    def _from_config(cls, config, **kwargs):
        """
        All context managers that the model should be initialized under go here.

        Args:
1295
1296
            torch_dtype (`torch.dtype`, *optional*):
                Override the default `torch.dtype` and load the model under this dtype.
1297
1298
        """
        torch_dtype = kwargs.pop("torch_dtype", None)
1299
        use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False)
1300
1301
1302
1303
1304
1305

        # override default dtype if needed
        dtype_orig = None
        if torch_dtype is not None:
            dtype_orig = cls._set_default_torch_dtype(torch_dtype)

1306
1307
1308
1309
1310
        config = copy.deepcopy(config)  # We do not want to modify the config inplace in _from_config.
        config._attn_implementation = kwargs.pop("attn_implementation", None)
        config = cls._autoset_attn_implementation(
            config, use_flash_attention_2=use_flash_attention_2, check_device_map=False
        )
1311

1312
1313
1314
1315
1316
1317
        if is_deepspeed_zero3_enabled():
            import deepspeed

            logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
            # this immediately partitions the model across all gpus, to avoid the overhead in time
            # and memory copying it on CPU or each GPU first
1318
            with deepspeed.zero.Init(config_dict_or_path=deepspeed_config()):
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
                model = cls(config, **kwargs)
        else:
            model = cls(config, **kwargs)

        # restore default dtype if it was modified
        if dtype_orig is not None:
            torch.set_default_dtype(dtype_orig)

        return model

1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
    @classmethod
    def _autoset_attn_implementation(
        cls,
        config,
        use_flash_attention_2: bool = False,
        torch_dtype: Optional[torch.dtype] = None,
        device_map: Optional[Union[str, Dict[str, int]]] = None,
        check_device_map: bool = True,
    ):
        """
        Automatically checks and dispatches to a default attention implementation. In order of priority:
            1. An implementation specified in `config._attn_implementation` (due for example to the argument attn_implementation="sdpa" in from_pretrained).
            2. DEPRECATED: if use_flash_attention_2 is set to `True` and `flash_attn` is available, flash attention. (`LlamaFlashAttention` for example)
            3. SDPA implementation, if available and supported by the model type. (`LlamaSdpaAttention` for example)
            4. The default model's implementation otherwise (`LlamaAttention` for example) .
        """
        # Here we use config._attn_implementation_internal to check whether the attention implementation was explicitely set by the user.
        # The property `PretrainedConfig._attn_implementation` is never `None`, for backward compatibility (always fall back on "eager").
        # The `hasattr` here is used as some Transformers tests for some reason do not call PretrainedConfig __init__ (e.g. test_no_super_init_config_and_model)
1348
        requested_attn_implementation = None
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
        if hasattr(config, "_attn_implementation_internal") and config._attn_implementation_internal is not None:
            if config._attn_implementation != "flash_attention_2" and use_flash_attention_2:
                raise ValueError(
                    f'Both attn_implementation="{config._attn_implementation}" and `use_flash_attention_2=True` were used when loading the model, which are not compatible.'
                    ' We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.'
                )

            if config._attn_implementation not in ["eager", "sdpa", "flash_attention_2"]:
                message = f'Specified `attn_implementation="{config._attn_implementation}"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)'
                if cls._supports_flash_attn_2:
                    message += ', `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)'
                if cls._supports_sdpa:
                    message += ', `"attn_implementation=sdpa"` (implementation using torch.nn.functional.scaled_dot_product_attention)'
                raise ValueError(message + ".")

            # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available.
1365
            requested_attn_implementation = config._attn_implementation_internal
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377

        if use_flash_attention_2:
            logger.warning_once(
                'The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.'
            )
            config._attn_implementation = "flash_attention_2"

        if config._attn_implementation == "flash_attention_2":
            cls._check_and_enable_flash_attn_2(
                config,
                torch_dtype=torch_dtype,
                device_map=device_map,
1378
                hard_check_only=False,
1379
1380
                check_device_map=check_device_map,
            )
1381
        elif requested_attn_implementation in [None, "sdpa"]:
1382
            # use_flash_attention_2 takes priority over SDPA, hence SDPA treated in this elif.
1383
1384
1385
1386
            config = cls._check_and_enable_sdpa(
                config, hard_check_only=False if requested_attn_implementation is None else True
            )
        else:
1387
1388
1389
1390
            config._attn_implementation = "eager"

        return config

1391
1392
1393
1394
1395
1396
1397
    @classmethod
    def _set_default_torch_dtype(cls, dtype: torch.dtype) -> torch.dtype:
        """
        Change the default dtype and return the previous one. This is needed when wanting to instantiate the model
        under specific dtype.

        Args:
1398
            dtype (`torch.dtype`):
1399
1400
1401
                a floating dtype to set to.

        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
1402
1403
            `torch.dtype`: the original `dtype` that can be used to restore `torch.set_default_dtype(dtype)` if it was
            modified. If it wasn't, returns `None`.
1404

1405
1406
        Note `set_default_dtype` currently only works with floating-point types and asserts if for example,
        `torch.int64` is passed. So if a non-float `dtype` is passed this functions will throw an exception.
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
        """
        if not dtype.is_floating_point:
            raise ValueError(
                f"Can't instantiate {cls.__name__} model under dtype={dtype} since it is not a floating point dtype"
            )

        logger.info(f"Instantiating {cls.__name__} model under default dtype {dtype}.")
        dtype_orig = torch.get_default_dtype()
        torch.set_default_dtype(dtype)
        return dtype_orig

1418
    @property
1419
1420
    def base_model(self) -> nn.Module:
        """
1421
        `torch.nn.Module`: The main body of the model.
1422
        """
1423
        return getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
1424

1425
1426
    @classmethod
    def can_generate(cls) -> bool:
1427
1428
1429
1430
1431
1432
        """
        Returns whether this model can generate sequences with `.generate()`.

        Returns:
            `bool`: Whether this model can generate sequences with `.generate()`.
        """
1433
1434
1435
        # Detects whether `prepare_inputs_for_generation` has been overwritten, which is a requirement for generation.
        # Alternativelly, the model can also have a custom `generate` function.
        if "GenerationMixin" in str(cls.prepare_inputs_for_generation) and "GenerationMixin" in str(cls.generate):
1436
1437
1438
            return False
        return True

1439
1440
    @classmethod
    def _check_and_enable_flash_attn_2(
1441
1442
1443
1444
1445
1446
        cls,
        config,
        torch_dtype: Optional[torch.dtype] = None,
        device_map: Optional[Union[str, Dict[str, int]]] = None,
        check_device_map: bool = True,
        hard_check_only: bool = False,
1447
1448
    ) -> PretrainedConfig:
        """
1449
        Checks the availability of Flash Attention 2 and compatibility with the current model.
1450

1451
        If all checks pass and `hard_check_only` is False, the method will set the config attribute `attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module.
1452
1453
1454
        """
        if not cls._supports_flash_attn_2:
            raise ValueError(
1455
1456
1457
                f"{cls.__name__} does not support Flash Attention 2.0 yet. Please request to add support where"
                f" the model is hosted, on its model hub page: https://huggingface.co/{config._name_or_path}/discussions/new"
                " or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/new"
1458
1459
            )

1460
        if not is_flash_attn_2_available():
1461
1462
1463
            preface = "FlashAttention2 has been toggled on, but it cannot be used due to the following error:"
            install_message = "Please refer to the documentation of https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashattention-2 to install Flash Attention 2."

1464
1465
1466
1467
1468
            if importlib.util.find_spec("flash_attn") is None:
                raise ImportError(f"{preface} the package flash_attn seems to be not installed. {install_message}")

            flash_attention_version = version.parse(importlib.metadata.version("flash_attn"))
            if torch.version.cuda:
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
                if flash_attention_version < version.parse("2.1.0"):
                    raise ImportError(
                        f"{preface} you need flash_attn package version to be greater or equal than 2.1.0. Detected version {flash_attention_version}. {install_message}"
                    )
                else:
                    raise ImportError(f"{preface} Flash Attention 2 is not available. {install_message}")
            elif torch.version.hip:
                if flash_attention_version < version.parse("2.0.4"):
                    raise ImportError(
                        f"{preface} you need flash_attn package version to be greater or equal than 2.0.4. Make sure to have that version installed - detected version {flash_attention_version}. {install_message}"
                    )
                else:
                    raise ImportError(f"{preface} Flash Attention 2 is not available. {install_message}")
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494

        _is_bettertransformer = getattr(cls, "use_bettertransformer", False)

        if _is_bettertransformer:
            raise ValueError(
                "Flash Attention 2 and BetterTransformer API are not compatible. Please make sure to disable BetterTransformers by doing model.reverse_bettertransformer()"
            )

        if torch_dtype is None:
            logger.warning(
                "You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour"
            )
        elif torch_dtype is not None and torch_dtype not in [torch.float16, torch.bfloat16]:
1495
1496
1497
            logger.warning(
                "Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes. "
                "No dtype was provided, you should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator."
1498
1499
            )

1500
1501
1502
        # The check `torch.empty(0).device.type != "cuda"` is needed as the model may be initialized after `torch.set_default_device` has been called,
        # or the model may be initialized under the context manager `with torch.device("cuda"):`.
        if check_device_map and device_map is None and torch.empty(0).device.type != "cuda":
1503
1504
            if torch.cuda.is_available():
                logger.warning(
1505
                    "You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU"
1506
1507
1508
1509
                    " after initializing it on CPU with `model.to('cuda')`."
                )
            else:
                raise ValueError(
1510
                    "You are attempting to use Flash Attention 2.0 with a model not initialized on GPU and with no GPU available. "
1511
1512
1513
1514
                    "This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map "
                    "or initialising the model on CPU and then moving it to GPU."
                )
        elif (
1515
1516
            check_device_map
            and device_map is not None
1517
1518
1519
1520
1521
1522
1523
            and isinstance(device_map, dict)
            and ("cpu" in device_map.values() or "disk" in device_map.values())
        ):
            raise ValueError(
                "You are attempting to use Flash Attention 2.0 with a model dispatched on CPU or disk. This is not supported. Please make sure to "
                "initialise the model on a GPU by passing a device_map that contains only GPU devices as keys."
            )
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
        if not hard_check_only:
            config._attn_implementation = "flash_attention_2"
        return config

    @classmethod
    def _check_and_enable_sdpa(cls, config, hard_check_only: bool = False) -> PretrainedConfig:
        """
        Checks the availability of SDPA for a given model.

        If all checks pass and `hard_check_only` is False, the method will set the config attribute `_attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module.
        """
        if hard_check_only:
            if not cls._supports_sdpa:
                raise ValueError(
                    f"{cls.__name__} does not support an attention implementation through torch.nn.functional.scaled_dot_product_attention yet. Please open an issue on GitHub to "
                    "request support for this architecture: https://github.com/huggingface/transformers/issues/new"
                )
            if not is_torch_sdpa_available():
                raise ImportError(
                    "PyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1."
                )

        if not is_torch_sdpa_available() or not cls._supports_sdpa:
            return config

        _is_bettertransformer = getattr(cls, "use_bettertransformer", False)
        if _is_bettertransformer:
            return config

        if not hard_check_only:
            config._attn_implementation = "sdpa"
1555
1556
        return config

1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
    def enable_input_require_grads(self):
        """
        Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping
        the model weights fixed.
        """

        def make_inputs_require_grads(module, input, output):
            output.requires_grad_(True)

        self._require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)

    def disable_input_require_grads(self):
        """
        Removes the `_require_grads_hook`.
        """
        self._require_grads_hook.remove()

1574
    def get_input_embeddings(self) -> nn.Module:
1575
1576
1577
1578
        """
        Returns the model's input embeddings.

        Returns:
1579
            `nn.Module`: A torch module mapping vocabulary to hidden states.
thomwolf's avatar
thomwolf committed
1580
        """
1581
        base_model = getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
1582
1583
1584
1585
        if base_model is not self:
            return base_model.get_input_embeddings()
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
1586

1587
    def set_input_embeddings(self, value: nn.Module):
1588
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
1589
        Set model's input embeddings.
1590
1591

        Args:
1592
            value (`nn.Module`): A module mapping vocabulary to hidden states.
thomwolf's avatar
thomwolf committed
1593
1594
1595
1596
1597
1598
        """
        base_model = getattr(self, self.base_model_prefix, self)
        if base_model is not self:
            base_model.set_input_embeddings(value)
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
1599

1600
    def get_output_embeddings(self) -> nn.Module:
1601
1602
1603
1604
        """
        Returns the model's output embeddings.

        Returns:
1605
            `nn.Module`: A torch module mapping hidden states to vocabulary.
thomwolf's avatar
thomwolf committed
1606
        """
1607
        return None  # Overwrite for models with output embeddings
thomwolf's avatar
thomwolf committed
1608

1609
1610
    def _init_weights(self, module):
        """
1611
1612
1613
1614
        Initialize the weights. This method should be overridden by derived class and is
        the only initialization method that will be called when loading a checkpoint
        using `from_pretrained`. Any attempt to initialize outside of this function
        will be useless as the torch.nn.init function are all replaced with skip.
1615
        """
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
        pass

    def _initialize_weights(self, module):
        """
        Initialize the weights if they are not already initialized.
        """
        if getattr(module, "_is_hf_initialized", False):
            return
        self._init_weights(module)
        module._is_hf_initialized = True
1626

1627
    def tie_weights(self):
1628
1629
        """
        Tie the weights between the input embeddings and the output embeddings.
1630

Sylvain Gugger's avatar
Sylvain Gugger committed
1631
1632
        If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the
        weights instead.
thomwolf's avatar
thomwolf committed
1633
        """
1634
1635
1636
1637
        if getattr(self.config, "tie_word_embeddings", True):
            output_embeddings = self.get_output_embeddings()
            if output_embeddings is not None:
                self._tie_or_clone_weights(output_embeddings, self.get_input_embeddings())
thomwolf's avatar
thomwolf committed
1638

1639
        if getattr(self.config, "is_encoder_decoder", False) and getattr(self.config, "tie_encoder_decoder", False):
Weizhen's avatar
Weizhen committed
1640
1641
            if hasattr(self, self.base_model_prefix):
                self = getattr(self, self.base_model_prefix)
1642
1643
            self._tie_encoder_decoder_weights(self.encoder, self.decoder, self.base_model_prefix)

Sylvain Gugger's avatar
Sylvain Gugger committed
1644
1645
1646
1647
        for module in self.modules():
            if hasattr(module, "_tie_weights"):
                module._tie_weights()

1648
1649
1650
    @staticmethod
    def _tie_encoder_decoder_weights(encoder: nn.Module, decoder: nn.Module, base_model_prefix: str):
        uninitialized_encoder_weights: List[str] = []
Weizhen's avatar
Weizhen committed
1651
1652
        if decoder.__class__ != encoder.__class__:
            logger.info(
Sylvain Gugger's avatar
Sylvain Gugger committed
1653
1654
                f"{decoder.__class__} and {encoder.__class__} are not equal. In this case make sure that all encoder"
                " weights are correctly initialized."
Weizhen's avatar
Weizhen committed
1655
            )
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665

        def tie_encoder_to_decoder_recursively(
            decoder_pointer: nn.Module,
            encoder_pointer: nn.Module,
            module_name: str,
            uninitialized_encoder_weights: List[str],
            depth=0,
        ):
            assert isinstance(decoder_pointer, nn.Module) and isinstance(
                encoder_pointer, nn.Module
1666
            ), f"{decoder_pointer} and {encoder_pointer} have to be of type nn.Module"
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
            if hasattr(decoder_pointer, "weight"):
                assert hasattr(encoder_pointer, "weight")
                encoder_pointer.weight = decoder_pointer.weight
                if hasattr(decoder_pointer, "bias"):
                    assert hasattr(encoder_pointer, "bias")
                    encoder_pointer.bias = decoder_pointer.bias
                return

            encoder_modules = encoder_pointer._modules
            decoder_modules = decoder_pointer._modules
            if len(decoder_modules) > 0:
                assert (
                    len(encoder_modules) > 0
                ), f"Encoder module {encoder_pointer} does not match decoder module {decoder_pointer}"

1682
                all_encoder_weights = {module_name + "/" + sub_name for sub_name in encoder_modules.keys()}
1683
1684
1685
1686
1687
                encoder_layer_pos = 0
                for name, module in decoder_modules.items():
                    if name.isdigit():
                        encoder_name = str(int(name) + encoder_layer_pos)
                        decoder_name = name
Weizhen's avatar
Weizhen committed
1688
1689
1690
                        if not isinstance(decoder_modules[decoder_name], type(encoder_modules[encoder_name])) and len(
                            encoder_modules
                        ) != len(decoder_modules):
1691
1692
                            # this can happen if the name corresponds to the position in a list module list of layers
                            # in this case the decoder has added a cross-attention that the encoder does not have
1693
                            # thus skip this step and subtract one layer pos from encoder
1694
1695
1696
1697
1698
1699
                            encoder_layer_pos -= 1
                            continue
                    elif name not in encoder_modules:
                        continue
                    elif depth > 500:
                        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1700
1701
                            "Max depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is"
                            " a circular dependency between two or more `nn.Modules` of your model."
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
                        )
                    else:
                        decoder_name = encoder_name = name
                    tie_encoder_to_decoder_recursively(
                        decoder_modules[decoder_name],
                        encoder_modules[encoder_name],
                        module_name + "/" + name,
                        uninitialized_encoder_weights,
                        depth=depth + 1,
                    )
                    all_encoder_weights.remove(module_name + "/" + encoder_name)

                uninitialized_encoder_weights += list(all_encoder_weights)

        # tie weights recursively
        tie_encoder_to_decoder_recursively(decoder, encoder, base_model_prefix, uninitialized_encoder_weights)
        if len(uninitialized_encoder_weights) > 0:
            logger.warning(
                f"The following encoder weights were not tied to the decoder {uninitialized_encoder_weights}"
            )

1723
    def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
Lysandre's avatar
Lysandre committed
1724
        """Tie or clone module weights depending of whether we are using TorchScript or not"""
thomwolf's avatar
thomwolf committed
1725
        if self.config.torchscript:
1726
            output_embeddings.weight = nn.Parameter(input_embeddings.weight.clone())
thomwolf's avatar
thomwolf committed
1727
        else:
1728
            output_embeddings.weight = input_embeddings.weight
thomwolf's avatar
thomwolf committed
1729

Sam Shleifer's avatar
Sam Shleifer committed
1730
        if getattr(output_embeddings, "bias", None) is not None:
1731
            output_embeddings.bias.data = nn.functional.pad(
1732
                output_embeddings.bias.data,
Lysandre's avatar
Lysandre committed
1733
1734
1735
1736
                (
                    0,
                    output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0],
                ),
1737
1738
                "constant",
                0,
1739
            )
1740
        if hasattr(output_embeddings, "out_features") and hasattr(input_embeddings, "num_embeddings"):
1741
            output_embeddings.out_features = input_embeddings.num_embeddings
1742

Marc Sun's avatar
Marc Sun committed
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
    def _get_no_split_modules(self, device_map: str):
        """
        Get the modules of the model that should not be spit when using device_map. We iterate through the modules to
        get the underlying `_no_split_modules`.

        Args:
            device_map (`str`):
                The device map value. Options are ["auto", "balanced", "balanced_low_0", "sequential"]

        Returns:
            `List[str]`: List of modules that should not be split
        """
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
        _no_split_modules = set()
        modules_to_check = [self]
        while len(modules_to_check) > 0:
            module = modules_to_check.pop(-1)
            # if the module does not appear in _no_split_modules, we also check the children
            if module.__class__.__name__ not in _no_split_modules:
                if isinstance(module, PreTrainedModel):
                    if module._no_split_modules is None:
                        raise ValueError(
                            f"{module.__class__.__name__} does not support `device_map='{device_map}'`. To implement support, the model "
                            "class needs to implement the `_no_split_modules` attribute."
                        )
                    else:
                        _no_split_modules = _no_split_modules | set(module._no_split_modules)
                modules_to_check += list(module.children())
Marc Sun's avatar
Marc Sun committed
1770
1771
        return list(_no_split_modules)

1772
1773
1774
    def resize_token_embeddings(
        self, new_num_tokens: Optional[int] = None, pad_to_multiple_of: Optional[int] = None
    ) -> nn.Embedding:
1775
        """
1776
        Resizes input token embeddings matrix of the model if `new_num_tokens != config.vocab_size`.
1777

1778
        Takes care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
thomwolf's avatar
thomwolf committed
1779

1780
        Arguments:
1781
            new_num_tokens (`int`, *optional*):
1782
                The new number of tokens in the embedding matrix. Increasing the size will add newly initialized
Sylvain Gugger's avatar
Sylvain Gugger committed
1783
1784
                vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just
                returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything.
1785
            pad_to_multiple_of (`int`, *optional*):
Arthur's avatar
Arthur committed
1786
1787
                If set will pad the embedding matrix to a multiple of the provided value.If `new_num_tokens` is set to
                `None` will just pad the embedding to a multiple of `pad_to_multiple_of`.
1788
1789
1790
1791
1792

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
                details about this, or help on choosing the correct value for resizing, refer to this guide:
                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc
1793
1794

        Return:
1795
            `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model.
thomwolf's avatar
thomwolf committed
1796
        """
1797
        model_embeds = self._resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
Arthur's avatar
Arthur committed
1798
        if new_num_tokens is None and pad_to_multiple_of is None:
thomwolf's avatar
thomwolf committed
1799
            return model_embeds
thomwolf's avatar
thomwolf committed
1800
1801

        # Update base model and current model config
Arthur's avatar
Arthur committed
1802
1803
        self.config.vocab_size = model_embeds.weight.shape[0]
        self.vocab_size = model_embeds.weight.shape[0]
thomwolf's avatar
thomwolf committed
1804
1805

        # Tie weights again if needed
1806
        self.tie_weights()
thomwolf's avatar
thomwolf committed
1807

thomwolf's avatar
thomwolf committed
1808
1809
        return model_embeds

1810
    def _resize_token_embeddings(self, new_num_tokens, pad_to_multiple_of=None):
thomwolf's avatar
thomwolf committed
1811
        old_embeddings = self.get_input_embeddings()
1812
        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens, pad_to_multiple_of)
1813
1814
1815
        if hasattr(old_embeddings, "_hf_hook"):
            hook = old_embeddings._hf_hook
            add_hook_to_module(new_embeddings, hook)
1816
1817
        old_embeddings_requires_grad = old_embeddings.weight.requires_grad
        new_embeddings.requires_grad_(old_embeddings_requires_grad)
thomwolf's avatar
thomwolf committed
1818
        self.set_input_embeddings(new_embeddings)
1819

1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
        # Update new_num_tokens with the actual size of new_embeddings
        if pad_to_multiple_of is not None:
            if is_deepspeed_zero3_enabled():
                import deepspeed

                with deepspeed.zero.GatheredParameters(new_embeddings.weight, modifier_rank=None):
                    new_num_tokens = new_embeddings.weight.shape[0]
            else:
                new_num_tokens = new_embeddings.weight.shape[0]

1830
1831
1832
        # if word embeddings are not tied, make sure that lm head is resized as well
        if self.get_output_embeddings() is not None and not self.config.tie_word_embeddings:
            old_lm_head = self.get_output_embeddings()
1833
            new_lm_head = self._get_resized_lm_head(old_lm_head, new_num_tokens)
1834
1835
1836
            if hasattr(old_lm_head, "_hf_hook"):
                hook = old_lm_head._hf_hook
                add_hook_to_module(new_lm_head, hook)
1837
1838
            old_lm_head_requires_grad = old_lm_head.weight.requires_grad
            new_lm_head.requires_grad_(old_lm_head_requires_grad)
1839
1840
            self.set_output_embeddings(new_lm_head)

thomwolf's avatar
thomwolf committed
1841
        return self.get_input_embeddings()
1842

1843
    def _get_resized_embeddings(
1844
1845
1846
1847
        self,
        old_embeddings: nn.Embedding,
        new_num_tokens: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
1848
    ) -> nn.Embedding:
1849
1850
1851
        """
        Build a resized Embedding Module from a provided token Embedding Module. Increasing the size will add newly
        initialized vectors at the end. Reducing the size will remove vectors from the end
1852
1853

        Args:
1854
            old_embeddings (`torch.nn.Embedding`):
1855
                Old embeddings to be resized.
1856
            new_num_tokens (`int`, *optional*):
1857
                New number of tokens in the embedding matrix.
1858
1859

                Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
1860
                vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
1861
                `torch.nn.Embedding` module of the model without doing anything.
1862
            pad_to_multiple_of (`int`, *optional*):
Arthur's avatar
Arthur committed
1863
1864
                If set will pad the embedding matrix to a multiple of the provided value. If `new_num_tokens` is set to
                `None` will just pad the embedding to a multiple of `pad_to_multiple_of`.
1865
1866
1867
1868
1869
1870

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
                details about this, or help on choosing the correct value for resizing, refer to this guide:
                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc

1871
1872

        Return:
1873
1874
            `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if
            `new_num_tokens` is `None`
1875
        """
1876
1877
1878
1879
1880
1881
1882
1883

        if pad_to_multiple_of is not None:
            if not isinstance(pad_to_multiple_of, int):
                raise ValueError(
                    f"Asking to pad the embedding matrix to a multiple of `{pad_to_multiple_of}`, which is not and integer. Please make sure to pass an integer"
                )
            if new_num_tokens is None:
                new_num_tokens = old_embeddings.weight.shape[0]
1884
            new_num_tokens = ((new_num_tokens + pad_to_multiple_of - 1) // pad_to_multiple_of) * pad_to_multiple_of
1885
        else:
1886
            logger.info(
1887
                "You are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding"
1888
                f" dimension will be {new_num_tokens}. This might induce some performance reduction as *Tensor Cores* will not be available."
1889
                " For more details about this, or help on choosing the correct value for resizing, refer to this guide:"
1890
1891
1892
                " https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc"
            )

1893
1894
1895
        if new_num_tokens is None:
            return old_embeddings

1896
1897
1898
1899
1900
1901
1902
1903
        if is_deepspeed_zero3_enabled():
            import deepspeed

            with deepspeed.zero.GatheredParameters(old_embeddings.weight, modifier_rank=None):
                old_num_tokens, old_embedding_dim = old_embeddings.weight.size()
        else:
            old_num_tokens, old_embedding_dim = old_embeddings.weight.size()

1904
        if old_num_tokens == new_num_tokens and not is_deepspeed_zero3_enabled():
1905
1906
            return old_embeddings

1907
1908
        if not isinstance(old_embeddings, nn.Embedding):
            raise TypeError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1909
1910
1911
                f"Old embeddings are of type {type(old_embeddings)}, which is not an instance of {nn.Embedding}. You"
                " should either use a different resize function or make sure that `old_embeddings` are an instance of"
                f" {nn.Embedding}."
1912
1913
            )

1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
        # Build new embeddings

        # When using DeepSpeed ZeRO-3, we shouldn't create new embeddings with DeepSpeed init
        # because the shape of the new embedding layer is used across various modeling files
        # as well as to update config vocab size. Shape will be 0 when using DeepSpeed init leading
        # to errors when training.
        new_embeddings = nn.Embedding(
            new_num_tokens,
            old_embedding_dim,
            device=old_embeddings.weight.device,
            dtype=old_embeddings.weight.dtype,
        )

        # initialize all new embeddings (in particular added tokens)
        self._init_weights(new_embeddings)

        # Copy token embeddings from the previous weights

1932
1933
        # numbers of tokens to copy
        n = min(old_num_tokens, new_num_tokens)
1934

1935
1936
1937
        if is_deepspeed_zero3_enabled():
            import deepspeed

1938
1939
1940
            params = [old_embeddings.weight, new_embeddings.weight]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
                new_embeddings.weight.data[:n, :] = old_embeddings.weight.data[:n, :]
1941
1942
        else:
            new_embeddings.weight.data[:n, :] = old_embeddings.weight.data[:n, :]
1943
1944
1945

        return new_embeddings

1946
    def _get_resized_lm_head(
1947
1948
        self, old_lm_head: nn.Linear, new_num_tokens: Optional[int] = None, transposed: Optional[bool] = False
    ) -> nn.Linear:
1949
1950
1951
1952
1953
        """
        Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized
        vectors at the end. Reducing the size will remove vectors from the end

        Args:
1954
            old_lm_head (`torch.nn.Linear`):
1955
                Old lm head liner layer to be resized.
1956
            new_num_tokens (`int`, *optional*):
1957
1958
1959
                New number of tokens in the linear matrix.

                Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
1960
                vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
1961
1962
1963
                `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults
                to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim,
                vocab_size` else `vocab_size, lm_head_dim`.
1964
1965

        Return:
Sylvain Gugger's avatar
Sylvain Gugger committed
1966
1967
            `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is
            `None`
1968
1969
1970
1971
        """
        if new_num_tokens is None:
            return old_lm_head

1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
        if is_deepspeed_zero3_enabled():
            import deepspeed

            with deepspeed.zero.GatheredParameters(old_lm_head.weight, modifier_rank=None):
                old_num_tokens, old_lm_head_dim = (
                    old_lm_head.weight.size() if not transposed else old_lm_head.weight.t().size()
                )
        else:
            old_num_tokens, old_lm_head_dim = (
                old_lm_head.weight.size() if not transposed else old_lm_head.weight.t().size()
            )
1983

1984
        if old_num_tokens == new_num_tokens and not is_deepspeed_zero3_enabled():
1985
1986
1987
1988
            return old_lm_head

        if not isinstance(old_lm_head, nn.Linear):
            raise TypeError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1989
1990
1991
                f"Old language model head is of type {type(old_lm_head)}, which is not an instance of {nn.Linear}. You"
                " should either use a different resize function or make sure that `old_lm_head` are an instance of"
                f" {nn.Linear}."
1992
1993
1994
1995
1996
1997
            )

        # Build new lm head
        new_lm_head_shape = (old_lm_head_dim, new_num_tokens) if not transposed else (new_num_tokens, old_lm_head_dim)
        has_new_lm_head_bias = old_lm_head.bias is not None

1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
        # When using DeepSpeed ZeRO-3, we shouldn't create new embeddings with DeepSpeed init
        # because the shape of the new embedding layer is used across various modeling files
        # as well as to update config vocab size. Shape will be 0 when using DeepSpeed init leading
        # to errors when training.
        new_lm_head = nn.Linear(
            *new_lm_head_shape,
            bias=has_new_lm_head_bias,
            device=old_lm_head.weight.device,
            dtype=old_lm_head.weight.dtype,
        )

        # initialize new lm head (in particular added tokens)
        self._init_weights(new_lm_head)

2012
2013
        num_tokens_to_copy = min(old_num_tokens, new_num_tokens)

2014
2015
2016
        if is_deepspeed_zero3_enabled():
            import deepspeed

2017
2018
            params = [old_lm_head.weight, old_lm_head.bias, new_lm_head.weight, new_lm_head.bias]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
2019
2020
2021
                self._copy_lm_head_original_to_resized(
                    new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
                )
2022
        else:
2023
2024
            self._copy_lm_head_original_to_resized(
                new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
2025
            )
2026
2027
2028

        return new_lm_head

2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
    def _copy_lm_head_original_to_resized(
        self, new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
    ):
        # Copy old lm head weights to new lm head
        if not transposed:
            new_lm_head.weight.data[:num_tokens_to_copy, :] = old_lm_head.weight.data[:num_tokens_to_copy, :]
        else:
            new_lm_head.weight.data[:, :num_tokens_to_copy] = old_lm_head.weight.data[:, :num_tokens_to_copy]

        # Copy bias weights to new lm head
        if has_new_lm_head_bias:
            new_lm_head.bias.data[:num_tokens_to_copy] = old_lm_head.bias.data[:num_tokens_to_copy]

2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
    def resize_position_embeddings(self, new_num_position_embeddings: int):
        raise NotImplementedError(
            f"`resize_position_embeddings` is not implemented for {self.__class__}`. To implement it, you should "
            f"overwrite this method in the class {self.__class__} in `modeling_{self.__class__.__module__}.py`"
        )

    def get_position_embeddings(self) -> Union[nn.Embedding, Tuple[nn.Embedding]]:
        raise NotImplementedError(
            f"`get_position_embeddings` is not implemented for {self.__class__}`. To implement it, you should "
            f"overwrite this method in the class {self.__class__} in `modeling_{self.__class__.__module__}.py`"
        )

2054
    def init_weights(self):
2055
        """
2056
2057
        If needed prunes and maybe initializes weights. If using a custom `PreTrainedModel`, you need to implement any
        initialization logic in `_init_weights`.
2058
        """
2059
2060
2061
2062
        # Prune heads if needed
        if self.config.pruned_heads:
            self.prune_heads(self.config.pruned_heads)

2063
2064
        if _init_weights:
            # Initialize weights
2065
            self.apply(self._initialize_weights)
2066
2067
2068
2069

            # Tie weights should be skipped when not initializing all weights
            # since from_pretrained(...) calls tie weights anyways
            self.tie_weights()
2070

2071
2072
2073
    def prune_heads(self, heads_to_prune: Dict[int, List[int]]):
        """
        Prunes heads of the base model.
2074

2075
        Arguments:
2076
            heads_to_prune (`Dict[int, List[int]]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2077
2078
2079
                Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads
                to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on
                layer 1 and heads 2 and 3 on layer 2.
thomwolf's avatar
thomwolf committed
2080
        """
2081
        # save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads
2082
        for layer, heads in heads_to_prune.items():
2083
2084
2085
            union_heads = set(self.config.pruned_heads.get(layer, [])) | set(heads)
            self.config.pruned_heads[layer] = list(union_heads)  # Unfortunately we have to store it as list for JSON

2086
        self.base_model._prune_heads(heads_to_prune)
thomwolf's avatar
thomwolf committed
2087

2088
    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
2089
2090
2091
2092
2093
        """
        Activates gradient checkpointing for the current model.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
2094
2095
2096
2097
2098
2099
2100

        We pass the `__call__` method of the modules instead of `forward` because `__call__` attaches all the hooks of
        the module. https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2

        Args:
            gradient_checkpointing_kwargs (dict, *optional*):
                Additional keyword arguments passed along to the `torch.utils.checkpoint.checkpoint` function.
2101
2102
2103
        """
        if not self.supports_gradient_checkpointing:
            raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
2104
2105
2106
2107

        if gradient_checkpointing_kwargs is None:
            gradient_checkpointing_kwargs = {}

2108
        gradient_checkpointing_func = functools.partial(checkpoint, **gradient_checkpointing_kwargs)
2109

2110
        # For old GC format (transformers < 4.35.0) for models that live on the Hub
Stas Bekman's avatar
Stas Bekman committed
2111
        # we will fall back to the overwritten `_set_gradient_checkpointing` method
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
        _is_using_old_format = "value" in inspect.signature(self._set_gradient_checkpointing).parameters

        if not _is_using_old_format:
            self._set_gradient_checkpointing(enable=True, gradient_checkpointing_func=gradient_checkpointing_func)
        else:
            self.apply(partial(self._set_gradient_checkpointing, value=True))
            logger.warn(
                "You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it)."
                "Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model."
            )
2122

2123
2124
2125
2126
2127
2128
2129
        if getattr(self, "_hf_peft_config_loaded", False):
            # When using PEFT + gradient checkpointing + Trainer we need to make sure the input has requires_grad=True
            # we do it also on PEFT: https://github.com/huggingface/peft/blob/85013987aa82aa1af3da1236b6902556ce3e483e/src/peft/peft_model.py#L334
            # When training with PEFT, only LoRA layers will have requires grad set to True, but the output of frozen layers need to propagate
            # the gradients to make sure the gradient flows.
            self.enable_input_require_grads()

2130
    def _set_gradient_checkpointing(self, enable: bool = True, gradient_checkpointing_func: Callable = checkpoint):
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
        is_gradient_checkpointing_set = False

        # Apply it on the top-level module in case the top-level modules supports it
        # for example, LongT5Stack inherits from `PreTrainedModel`.
        if hasattr(self, "gradient_checkpointing"):
            self._gradient_checkpointing_func = gradient_checkpointing_func
            self.gradient_checkpointing = enable
            is_gradient_checkpointing_set = True

        for module in self.modules():
            if hasattr(module, "gradient_checkpointing"):
                module._gradient_checkpointing_func = gradient_checkpointing_func
                module.gradient_checkpointing = enable
                is_gradient_checkpointing_set = True

        if not is_gradient_checkpointing_set:
            raise ValueError(
                f"{self.__class__.__name__} is not compatible with gradient checkpointing. Make sure all the architecture support it by setting a boolean attribute"
                " `gradient_checkpointing` to modules of the model that uses checkpointing."
            )

2152
    def gradient_checkpointing_disable(self):
2153
2154
2155
2156
2157
2158
2159
        """
        Deactivates gradient checkpointing for the current model.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        if self.supports_gradient_checkpointing:
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
            # For old GC format (transformers < 4.35.0) for models that live on the Hub
            # we will fall back to the overwritten `_set_gradient_checkpointing` methid
            _is_using_old_format = "value" in inspect.signature(self._set_gradient_checkpointing).parameters
            if not _is_using_old_format:
                self._set_gradient_checkpointing(enable=False)
            else:
                logger.warn(
                    "You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it)."
                    "Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model."
                )
                self.apply(partial(self._set_gradient_checkpointing, value=False))
2171

2172
2173
2174
        if getattr(self, "_hf_peft_config_loaded", False):
            self.disable_input_require_grads()

2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
    @property
    def is_gradient_checkpointing(self) -> bool:
        """
        Whether gradient checkpointing is activated for this model or not.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())

2185
2186
2187
    def save_pretrained(
        self,
        save_directory: Union[str, os.PathLike],
2188
        is_main_process: bool = True,
2189
2190
        state_dict: Optional[dict] = None,
        save_function: Callable = torch.save,
Sylvain Gugger's avatar
Sylvain Gugger committed
2191
        push_to_hub: bool = False,
2192
        max_shard_size: Union[int, str] = "5GB",
2193
        safe_serialization: bool = True,
2194
        variant: Optional[str] = None,
2195
        token: Optional[Union[str, bool]] = None,
2196
        save_peft_format: bool = True,
Sylvain Gugger's avatar
Sylvain Gugger committed
2197
        **kwargs,
2198
    ):
2199
2200
        """
        Save a model and its configuration file to a directory, so that it can be re-loaded using the
2201
        [`~PreTrainedModel.from_pretrained`] class method.
2202

2203
        Arguments:
2204
            save_directory (`str` or `os.PathLike`):
2205
                Directory to which to save. Will be created if it doesn't exist.
2206
2207
2208
2209
            is_main_process (`bool`, *optional*, defaults to `True`):
                Whether the process calling this is the main process or not. Useful when in distributed training like
                TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on
                the main process to avoid race conditions.
2210
            state_dict (nested dictionary of `torch.Tensor`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2211
2212
2213
                The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only
                save parts of the model or if special precautions need to be taken when recovering the state dictionary
                of a model (like when using model parallelism).
2214
            save_function (`Callable`):
2215
                The function to use to save the state dictionary. Useful on distributed training like TPUs when one
2216
2217
                need to replace `torch.save` by another method.
            push_to_hub (`bool`, *optional*, defaults to `False`):
2218
2219
2220
                Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
                repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
                namespace).
2221
            max_shard_size (`int` or `str`, *optional*, defaults to `"5GB"`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2222
2223
                The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
                lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
2224
2225
                We default it to 5GB in order for models to be able to run easily on free-tier google colab instances
                without CPU OOM issues.
Sylvain Gugger's avatar
Sylvain Gugger committed
2226
2227
2228
2229
2230
2231
2232
2233

                <Tip warning={true}>

                If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
                which will be bigger than `max_shard_size`.

                </Tip>

2234
            safe_serialization (`bool`, *optional*, defaults to `True`):
2235
                Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
2236
2237
            variant (`str`, *optional*):
                If specified, weights are saved in the format pytorch_model.<variant>.bin.
2238
2239
2240
            token (`str` or `bool`, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
2241
2242
2243
2244
            save_peft_format (`bool`, *optional*, defaults to `True`):
                For backward compatibility with PEFT library, in case adapter weights are attached to the model, all
                keys of the state dict of adapters needs to be pre-pended with `base_model.model`. Advanced users can
                disable this behaviours by setting `save_peft_format` to `False`.
2245
            kwargs (`Dict[str, Any]`, *optional*):
2246
                Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
2247
        """
2248
        use_auth_token = kwargs.pop("use_auth_token", None)
2249
        ignore_metadata_errors = kwargs.pop("ignore_metadata_errors", False)
2250
2251
2252

        if use_auth_token is not None:
            warnings.warn(
2253
2254
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        if token is not None:
            kwargs["token"] = token

Younes Belkada's avatar
Younes Belkada committed
2265
2266
        _hf_peft_config_loaded = getattr(self, "_hf_peft_config_loaded", False)

2267
        # Checks if the model has been loaded in 8-bit
Younes Belkada's avatar
Younes Belkada committed
2268
2269
2270
2271
2272
        if (
            getattr(self, "is_loaded_in_8bit", False)
            and not getattr(self, "is_8bit_serializable", False)
            and not _hf_peft_config_loaded
        ):
2273
2274
2275
            raise NotImplementedError(
                "You are calling `save_pretrained` to a 8-bit converted model, but your `bitsandbytes` version doesn't support it. "
                "If you want to save 8-bit models, make sure to have `bitsandbytes>0.37.2` installed."
2276
2277
            )

2278
2279
2280
2281
2282
        if (
            getattr(self, "is_loaded_in_4bit", False)
            and not getattr(self, "is_4bit_serializable", False)
            and not _hf_peft_config_loaded
        ):
2283
            raise NotImplementedError(
2284
2285
                "You are calling `save_pretrained` to a 4-bit converted model, but your `bitsandbytes` version doesn't support it. "
                "If you want to save 4-bit models, make sure to have `bitsandbytes>=0.41.3` installed."
2286
2287
            )

2288
2289
2290
        if getattr(self, "_awq_is_fused", False):
            raise ValueError("You cannot save an AWQ model that uses fused modules!")

2291
2292
2293
2294
2295
        if "save_config" in kwargs:
            warnings.warn(
                "`save_config` is deprecated and will be removed in v5 of Transformers. Use `is_main_process` instead."
            )
            is_main_process = kwargs.pop("save_config")
2296
2297
        if safe_serialization and not is_safetensors_available():
            raise ImportError("`safe_serialization` requires the `safetensors library: `pip install safetensors`.")
2298

2299
        if os.path.isfile(save_directory):
2300
            logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
2301
            return
2302

2303
2304
        os.makedirs(save_directory, exist_ok=True)

2305
2306
        if push_to_hub:
            commit_message = kwargs.pop("commit_message", None)
2307
            repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
2308
            repo_id = self._create_repo(repo_id, **kwargs)
2309
            files_timestamps = self._get_files_timestamps(save_directory)
2310

Julien Chaumond's avatar
Julien Chaumond committed
2311
        # Only save the model itself if we are using distributed training
2312
        model_to_save = unwrap_model(self)
2313

2314
2315
2316
2317
2318
        # save the string version of dtype to the config, e.g. convert torch.float32 => "float32"
        # we currently don't use this setting automatically, but may start to use with v5
        dtype = get_parameter_dtype(model_to_save)
        model_to_save.config.torch_dtype = str(dtype).split(".")[1]

Julien Chaumond's avatar
Julien Chaumond committed
2319
2320
2321
        # Attach architecture to the config
        model_to_save.config.architectures = [model_to_save.__class__.__name__]

2322
2323
2324
2325
2326
        # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be
        # loaded from the Hub.
        if self._auto_class is not None:
            custom_object_save(self, save_directory, config=self.config)

2327
        # Save the config
2328
        if is_main_process:
2329
2330
            if not _hf_peft_config_loaded:
                model_to_save.config.save_pretrained(save_directory)
2331
2332
            if self.can_generate():
                model_to_save.generation_config.save_pretrained(save_directory)
2333

2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
            if _hf_peft_config_loaded:
                logger.info(
                    "Detected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved."
                )
                state_dict = model_to_save.get_adapter_state_dict()

                if save_peft_format:
                    logger.info(
                        "To match the expected format of the PEFT library, all keys of the state dict of adapters will be pre-pended with `base_model.model`."
                    )
                    peft_state_dict = {}
                    for key, value in state_dict.items():
                        peft_state_dict[f"base_model.model.{key}"] = value
                    state_dict = peft_state_dict

2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
                active_adapter = self.active_adapters()

                if len(active_adapter) > 1:
                    raise ValueError(
                        "Multiple active adapters detected, saving multiple active adapters is not supported yet. You can save adapters separately one by one "
                        "by iteratively calling `model.set_adapter(adapter_name)` then `model.save_pretrained(...)`"
                    )
                active_adapter = active_adapter[0]

                current_peft_config = self.peft_config[active_adapter]
2359
2360
                current_peft_config.save_pretrained(save_directory)

2361
2362
2363
        # Save the model
        if state_dict is None:
            state_dict = model_to_save.state_dict()
2364

2365
2366
2367
2368
2369
        # Translate state_dict from smp to hf if saving with smp >= 1.10
        if IS_SAGEMAKER_MP_POST_1_10:
            for smp_to_hf, _ in smp.state.module_manager.translate_functions:
                state_dict = smp_to_hf(state_dict)

2370
        # Handle the case where some state_dict keys shouldn't be saved
2371
        if self._keys_to_ignore_on_save is not None:
2372
            for ignore_key in self._keys_to_ignore_on_save:
2373
2374
                if ignore_key in state_dict.keys():
                    del state_dict[ignore_key]
2375
2376
2377
2378
2379
        if safe_serialization:
            # Safetensors does not allow tensor aliasing.
            # We're going to remove aliases before saving
            ptrs = collections.defaultdict(list)
            for name, tensor in state_dict.items():
2380
2381
2382
2383
2384
2385
2386
                # Sometimes in the state_dict we have non-tensor objects.
                # e.g. in bitsandbytes we have some `str` objects in the state_dict
                if isinstance(tensor, torch.Tensor):
                    ptrs[id_tensor_storage(tensor)].append(name)
                else:
                    # In the non-tensor case, fall back to the pointer of the object itself
                    ptrs[id(tensor)].append(name)
2387
2388
2389
2390
2391
2392
2393

            # These are all the pointers of shared tensors.
            shared_ptrs = {ptr: names for ptr, names in ptrs.items() if len(names) > 1}
            warn_names = set()
            for names in shared_ptrs.values():
                # Removing the keys which are declared as known duplicates on
                # load. This allows to make sure the name which is kept is consistent.
Sylvain Gugger's avatar
Sylvain Gugger committed
2394
                if self._tied_weights_keys is not None:
2395
2396
                    found = 0
                    for name in sorted(names):
Sylvain Gugger's avatar
Sylvain Gugger committed
2397
                        matches_pattern = any(re.search(pat, name) for pat in self._tied_weights_keys)
2398
                        if matches_pattern and name in state_dict:
2399
2400
2401
                            found += 1
                            if found < len(names):
                                del state_dict[name]
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418

                # When not all duplicates have been cleaned, still remove those keys, but put a clear warning.
                # If the link between tensors was done at runtime then `from_pretrained` will not get
                # the key back leading to random tensor. A proper warning will be shown
                # during reload (if applicable), but since the file is not necessarily compatible with
                # the config, better show a proper warning.
                found = 0
                for name in names:
                    if name in state_dict:
                        found += 1
                        if found > 1:
                            del state_dict[name]
                            warn_names.add(name)
            if len(warn_names) > 0:
                logger.warning_once(
                    f"Removed shared tensor {warn_names} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading",
                )
2419

Sylvain Gugger's avatar
Sylvain Gugger committed
2420
        # Shard the model if it is too big.
2421
2422
2423
2424
2425
        if not _hf_peft_config_loaded:
            weights_name = SAFE_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME
            weights_name = _add_variant(weights_name, variant)
        else:
            weights_name = ADAPTER_SAFE_WEIGHTS_NAME if safe_serialization else ADAPTER_WEIGHTS_NAME
2426

2427
        shards, index = shard_checkpoint(state_dict, max_shard_size=max_shard_size, weights_name=weights_name)
Sylvain Gugger's avatar
Sylvain Gugger committed
2428
2429
2430
2431

        # Clean the folder from a previous save
        for filename in os.listdir(save_directory):
            full_filename = os.path.join(save_directory, filename)
2432
2433
            # If we have a shard file that is not going to be replaced, we delete it, but only from the main process
            # in distributed settings to avoid race conditions.
2434
            weights_no_suffix = weights_name.replace(".bin", "").replace(".safetensors", "")
2435
2436
2437

            # make sure that file to be deleted matches format of sharded file, e.g. pytorch_model-00001-of-00005
            filename_no_suffix = filename.replace(".bin", "").replace(".safetensors", "")
2438
            reg = re.compile(r"(.*?)-\d{5}-of-\d{5}")
2439

2440
            if (
2441
                filename.startswith(weights_no_suffix)
2442
2443
2444
                and os.path.isfile(full_filename)
                and filename not in shards.keys()
                and is_main_process
2445
                and reg.fullmatch(filename_no_suffix) is not None
2446
            ):
Sylvain Gugger's avatar
Sylvain Gugger committed
2447
                os.remove(full_filename)
2448

Sylvain Gugger's avatar
Sylvain Gugger committed
2449
2450
        # Save the model
        for shard_file, shard in shards.items():
2451
2452
2453
2454
2455
2456
            if safe_serialization:
                # At some point we will need to deal better with save_function (used for TPU and other distributed
                # joyfulness), but for now this enough.
                safe_save_file(shard, os.path.join(save_directory, shard_file), metadata={"format": "pt"})
            else:
                save_function(shard, os.path.join(save_directory, shard_file))
Sylvain Gugger's avatar
Sylvain Gugger committed
2457
2458

        if index is None:
2459
2460
            weights_file_name = SAFE_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME
            path_to_weights = os.path.join(save_directory, _add_variant(weights_file_name, variant))
2461
            logger.info(f"Model weights saved in {path_to_weights}")
Sylvain Gugger's avatar
Sylvain Gugger committed
2462
        else:
2463
            save_index_file = SAFE_WEIGHTS_INDEX_NAME if safe_serialization else WEIGHTS_INDEX_NAME
2464
            save_index_file = os.path.join(save_directory, _add_variant(save_index_file, variant))
Sylvain Gugger's avatar
Sylvain Gugger committed
2465
2466
2467
2468
2469
2470
2471
2472
2473
            # Save the index as well
            with open(save_index_file, "w", encoding="utf-8") as f:
                content = json.dumps(index, indent=2, sort_keys=True) + "\n"
                f.write(content)
            logger.info(
                f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be "
                f"split in {len(shards)} checkpoint shards. You can find where each parameters has been saved in the "
                f"index located at {save_index_file}."
            )
2474

Sylvain Gugger's avatar
Sylvain Gugger committed
2475
        if push_to_hub:
2476
2477
2478
2479
2480
2481
2482
2483
            # Eventually create an empty model card
            model_card = create_and_tag_model_card(
                repo_id, self.model_tags, token=token, ignore_metadata_errors=ignore_metadata_errors
            )

            # Update model card if needed:
            model_card.save(os.path.join(save_directory, "README.md"))

2484
            self._upload_modified_files(
2485
2486
2487
2488
                save_directory,
                repo_id,
                files_timestamps,
                commit_message=commit_message,
2489
                token=token,
2490
            )
Sylvain Gugger's avatar
Sylvain Gugger committed
2491

2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
    @wraps(PushToHubMixin.push_to_hub)
    def push_to_hub(self, *args, **kwargs):
        tags = self.model_tags if self.model_tags is not None else []

        tags_kwargs = kwargs.get("tags", [])
        if isinstance(tags_kwargs, str):
            tags_kwargs = [tags_kwargs]

        for tag in tags_kwargs:
            if tag not in tags:
                tags.append(tag)

        if tags:
            kwargs["tags"] = tags
        return super().push_to_hub(*args, **kwargs)

2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
    def get_memory_footprint(self, return_buffers=True):
        r"""
        Get the memory footprint of a model. This will return the memory footprint of the current model in bytes.
        Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the
        PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2

        Arguments:
            return_buffers (`bool`, *optional*, defaults to `True`):
                Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers
                are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch
                norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2
        """
        mem = sum([param.nelement() * param.element_size() for param in self.parameters()])
        if return_buffers:
            mem_bufs = sum([buf.nelement() * buf.element_size() for buf in self.buffers()])
            mem = mem + mem_bufs
        return mem

2526
    @wraps(torch.nn.Module.cuda)
2527
2528
    def cuda(self, *args, **kwargs):
        # Checks if the model has been loaded in 8-bit
Marc Sun's avatar
Marc Sun committed
2529
        if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
2530
2531
2532
2533
2534
2535
2536
            raise ValueError(
                "Calling `cuda()` is not supported for `4-bit` or `8-bit` quantized models. Please use the model as it is, since the"
                " model has already been set to the correct devices and casted to the correct `dtype`."
            )
        else:
            return super().cuda(*args, **kwargs)

2537
    @wraps(torch.nn.Module.to)
2538
2539
    def to(self, *args, **kwargs):
        # Checks if the model has been loaded in 8-bit
Marc Sun's avatar
Marc Sun committed
2540
        if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
2541
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2542
                "`.to` is not supported for `4-bit` or `8-bit` bitsandbytes models. Please use the model as it is, since the"
2543
2544
                " model has already been set to the correct devices and casted to the correct `dtype`."
            )
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
        elif getattr(self, "quantization_method", None) == QuantizationMethod.GPTQ:
            # For GPTQ models, we prevent users from casting the model to another dytpe to restrict unwanted behaviours.
            # the correct API should be to load the model with the desired dtype directly through `from_pretrained`.
            dtype_present_in_args = False

            if "dtype" not in kwargs:
                for arg in args:
                    if isinstance(arg, torch.dtype):
                        dtype_present_in_args = True
                        break
            else:
                dtype_present_in_args = True

            if dtype_present_in_args:
                raise ValueError(
                    "You cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired"
                    " `dtype` by passing the correct `torch_dtype` argument."
                )
        return super().to(*args, **kwargs)
2564
2565

    def half(self, *args):
Marc Sun's avatar
Marc Sun committed
2566
        # Checks if the model is quantized
2567
        if getattr(self, "is_quantized", False):
2568
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2569
                "`.half()` is not supported for quantized model. Please use the model as it is, since the"
2570
2571
2572
2573
2574
2575
                " model has already been casted to the correct `dtype`."
            )
        else:
            return super().half(*args)

    def float(self, *args):
Marc Sun's avatar
Marc Sun committed
2576
        # Checks if the model is quantized
2577
        if getattr(self, "is_quantized", False):
2578
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2579
                "`.float()` is not supported for quantized model. Please use the model as it is, since the"
2580
2581
2582
2583
2584
                " model has already been casted to the correct `dtype`."
            )
        else:
            return super().float(*args)

2585
    @classmethod
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
        *model_args,
        config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
        cache_dir: Optional[Union[str, os.PathLike]] = None,
        ignore_mismatched_sizes: bool = False,
        force_download: bool = False,
        local_files_only: bool = False,
        token: Optional[Union[str, bool]] = None,
        revision: str = "main",
        use_safetensors: bool = None,
        **kwargs,
    ):
2600
2601
        r"""
        Instantiate a pretrained pytorch model from a pre-trained model configuration.
2602

Sylvain Gugger's avatar
Sylvain Gugger committed
2603
2604
        The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
        the model, you should first set it back in training mode with `model.train()`.
2605

2606
        The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come
2607
2608
        pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning
        task.
2609

2610
        The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those
2611
        weights are discarded.
2612

2613
        Parameters:
2614
            pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
2615
2616
                Can be either:

2617
                    - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
Sylvain Gugger's avatar
Sylvain Gugger committed
2618
2619
                      Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
                      user or organization name, like `dbmdz/bert-base-german-cased`.
2620
2621
2622
                    - A path to a *directory* containing model weights saved using
                      [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
                    - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
Sylvain Gugger's avatar
Sylvain Gugger committed
2623
2624
2625
                      this case, `from_tf` should be set to `True` and a configuration object should be provided as
                      `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
                      PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
2626
                    - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g,
Sylvain Gugger's avatar
Sylvain Gugger committed
2627
2628
                      `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to
                      `True`.
2629
2630
2631
2632
2633
                    - `None` if you are both providing the configuration and state dictionary (resp. with keyword
                      arguments `config` and `state_dict`).
            model_args (sequence of positional arguments, *optional*):
                All remaining positional arguments will be passed to the underlying model's `__init__` method.
            config (`Union[PretrainedConfig, str, os.PathLike]`, *optional*):
2634
2635
                Can be either:

2636
2637
                    - an instance of a class derived from [`PretrainedConfig`],
                    - a string or path valid as input to [`~PretrainedConfig.from_pretrained`].
2638

2639
                Configuration for the model to use instead of an automatically loaded configuration. Configuration can
2640
2641
                be automatically loaded when:

2642
                    - The model is a model provided by the library (loaded with the *model id* string of a pretrained
2643
                      model).
Sylvain Gugger's avatar
Sylvain Gugger committed
2644
2645
                    - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
                      save directory.
2646
2647
2648
                    - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
                      configuration JSON file named *config.json* is found in the directory.
            state_dict (`Dict[str, torch.Tensor]`, *optional*):
2649
2650
2651
                A state dictionary to use instead of a state dictionary loaded from saved weights file.

                This option can be used if you want to create a model from a pretrained configuration but load your own
Sylvain Gugger's avatar
Sylvain Gugger committed
2652
                weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and
2653
2654
                [`~PreTrainedModel.from_pretrained`] is not a simpler option.
            cache_dir (`Union[str, os.PathLike]`, *optional*):
2655
2656
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
2657
            from_tf (`bool`, *optional*, defaults to `False`):
2658
                Load the model weights from a TensorFlow checkpoint save file (see docstring of
2659
2660
                `pretrained_model_name_or_path` argument).
            from_flax (`bool`, *optional*, defaults to `False`):
2661
                Load the model weights from a Flax checkpoint save file (see docstring of
2662
2663
                `pretrained_model_name_or_path` argument).
            ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`):
2664
2665
2666
                Whether or not to raise an error if some of the weights from the checkpoint do not have the same size
                as the weights of the model (if for instance, you are instantiating a model with 10 labels from a
                checkpoint with 3 labels).
2667
            force_download (`bool`, *optional*, defaults to `False`):
2668
2669
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
2670
            resume_download (`bool`, *optional*, defaults to `False`):
2671
2672
                Whether or not to delete incompletely received files. Will attempt to resume the download if such a
                file exists.
2673
            proxies (`Dict[str, str]`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
2674
2675
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
2676
            output_loading_info(`bool`, *optional*, defaults to `False`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2677
                Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
2678
            local_files_only(`bool`, *optional*, defaults to `False`):
Stas Bekman's avatar
Stas Bekman committed
2679
                Whether or not to only look at local files (i.e., do not try to download the model).
2680
            token (`str` or `bool`, *optional*):
2681
2682
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
2683
            revision (`str`, *optional*, defaults to `"main"`):
Julien Chaumond's avatar
Julien Chaumond committed
2684
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
2685
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
Julien Chaumond's avatar
Julien Chaumond committed
2686
                identifier allowed by git.
2687
2688
2689
2690
2691
2692
2693

                <Tip>

                To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>".

                </Tip>

2694
            mirror (`str`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
2695
2696
2697
                Mirror source to accelerate downloads in China. If you are from China and have an accessibility
                problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety.
                Please refer to the mirror site for more information.
2698
            _fast_init(`bool`, *optional*, defaults to `True`):
2699
2700
                Whether or not to disable fast initialization.

2701
2702
                <Tip warning={true}>

Sylvain Gugger's avatar
Sylvain Gugger committed
2703
2704
2705
                One should only disable *_fast_init* to ensure backwards compatibility with `transformers.__version__ <
                4.6.0` for seeded model initialization. This argument will be removed at the next major version. See
                [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information.
2706

2707
                </Tip>
2708

2709
2710
2711
            > Parameters for big model inference

            low_cpu_mem_usage(`bool`, *optional*):
2712
2713
2714
                Tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
                This is an experimental feature and a subject to change at any moment.
            torch_dtype (`str` or `torch.dtype`, *optional*):
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
                Override the default `torch.dtype` and load the model under a specific `dtype`. The different options
                are:

                1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified
                  `dtype`, ignoring the model's `config.torch_dtype` if one exists. If not specified
                  - the model will get loaded in `torch.float` (fp32).

                2. `"auto"` - A `torch_dtype` entry in the `config.json` file of the model will be
                  attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in
                  the checkpoint that's of a floating point type and use that as `dtype`. This will load the model
                  using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how
                  the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32.

                <Tip>

                For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or
                reach out to the authors and ask them to add this information to the model's card and to insert the
                `torch_dtype` entry in `config.json` on the hub.

                </Tip>

2736
            device_map (`str` or `Dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*):
2737
2738
                A map that specifies where each submodule should go. It doesn't need to be refined to each
                parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the
2739
2740
2741
                same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank
                like `1`) on which the model will be allocated, the device map will map the entire model to this
                device. Passing `device_map = 0` means put the whole model on GPU 0.
2742

2743
2744
                To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For
                more information about each option see [designing a device
Patrick von Platen's avatar
Patrick von Platen committed
2745
                map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
2746
2747
2748
            max_memory (`Dict`, *optional*):
                A dictionary device identifier to maximum memory. Will default to the maximum memory available for each
                GPU and the available CPU RAM if unset.
2749
2750
            offload_folder (`str` or `os.PathLike`, *optional*):
                If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
2751
            offload_state_dict (`bool`, *optional*):
2752
                If `True`, will temporarily offload the CPU state dict to the hard drive to avoid getting out of CPU
2753
2754
                RAM if the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to
                `True` when there is some disk offload.
2755
2756
            load_in_8bit (`bool`, *optional*, defaults to `False`):
                If `True`, will convert the loaded model into mixed-8bit quantized model. To use this feature please
2757
2758
2759
2760
                install `bitsandbytes` (`pip install -U bitsandbytes`).
            load_in_4bit (`bool`, *optional*, defaults to `False`):
                If `True`, will convert the loaded model into 4bit precision quantized model. To use this feature
                install the latest version of `bitsandbytes` (`pip install -U bitsandbytes`).
Marc Sun's avatar
Marc Sun committed
2761
2762
2763
            quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*):
                A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g
                bitsandbytes, gptq)
2764
2765
2766
            subfolder (`str`, *optional*, defaults to `""`):
                In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
                specify the folder name here.
2767
2768
2769
            variant (`str`, *optional*):
                If specified load weights from `variant` filename, *e.g.* pytorch_model.<variant>.bin. `variant` is
                ignored when using `from_tf` or `from_flax`.
2770
2771
2772
            use_safetensors (`bool`, *optional*, defaults to `None`):
                Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors`
                is not installed, it will be set to `False`.
2773

2774
            kwargs (remaining dictionary of keyword arguments, *optional*):
2775
                Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
2776
                `output_attentions=True`). Behaves differently depending on whether a `config` is provided or
2777
2778
                automatically loaded:

2779
2780
                    - If a configuration is provided with `config`, `**kwargs` will be directly passed to the
                      underlying model's `__init__` method (we assume all relevant updates to the configuration have
2781
                      already been done)
2782
                    - If a configuration is not provided, `kwargs` will be first passed to the configuration class
Sylvain Gugger's avatar
Sylvain Gugger committed
2783
2784
2785
2786
                      initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
                      corresponds to a configuration attribute will be used to override said attribute with the
                      supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
                      will be passed to the underlying model's `__init__` function.
2787
2788
2789

        <Tip>

Sylvain Gugger's avatar
Sylvain Gugger committed
2790
2791
        Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to
        use this method in a firewalled environment.
2792
2793
2794
2795
2796
2797
2798

        </Tip>

        Examples:

        ```python
        >>> from transformers import BertConfig, BertModel
Sylvain Gugger's avatar
Sylvain Gugger committed
2799

2800
        >>> # Download model and configuration from huggingface.co and cache.
Sylvain Gugger's avatar
Sylvain Gugger committed
2801
        >>> model = BertModel.from_pretrained("bert-base-uncased")
2802
        >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable).
Sylvain Gugger's avatar
Sylvain Gugger committed
2803
        >>> model = BertModel.from_pretrained("./test/saved_model/")
2804
        >>> # Update configuration during loading.
Sylvain Gugger's avatar
Sylvain Gugger committed
2805
        >>> model = BertModel.from_pretrained("bert-base-uncased", output_attentions=True)
2806
2807
        >>> assert model.config.output_attentions == True
        >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable).
Sylvain Gugger's avatar
Sylvain Gugger committed
2808
2809
        >>> config = BertConfig.from_json_file("./tf_model/my_tf_model_config.json")
        >>> model = BertModel.from_pretrained("./tf_model/my_tf_checkpoint.ckpt.index", from_tf=True, config=config)
2810
        >>> # Loading from a Flax checkpoint file instead of a PyTorch model (slower)
Sylvain Gugger's avatar
Sylvain Gugger committed
2811
        >>> model = BertModel.from_pretrained("bert-base-uncased", from_flax=True)
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
        ```

        * `low_cpu_mem_usage` algorithm:

        This is an experimental function that loads the model using ~1x model size CPU memory

        Here is how it works:

        1. save which state_dict keys we have
        2. drop state_dict before the model is created, since the latter takes 1x model size CPU memory
        3. after the model has been instantiated switch to the meta device all params/buffers that
        are going to be replaced from the loaded state_dict
        4. load state_dict 2nd time
        5. replace the params/buffers from the state_dict

        Currently, it can't handle deepspeed ZeRO stage 3 and ignores loading errors

        """
2830
2831
        state_dict = kwargs.pop("state_dict", None)
        from_tf = kwargs.pop("from_tf", False)
2832
        from_flax = kwargs.pop("from_flax", False)
2833
2834
2835
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        output_loading_info = kwargs.pop("output_loading_info", False)
2836
        use_auth_token = kwargs.pop("use_auth_token", None)
2837
        trust_remote_code = kwargs.pop("trust_remote_code", None)
Sylvain Gugger's avatar
Sylvain Gugger committed
2838
        _ = kwargs.pop("mirror", None)
2839
2840
        from_pipeline = kwargs.pop("_from_pipeline", None)
        from_auto_class = kwargs.pop("_from_auto", False)
2841
        _fast_init = kwargs.pop("_fast_init", True)
2842
        torch_dtype = kwargs.pop("torch_dtype", None)
2843
2844
        low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", None)
        device_map = kwargs.pop("device_map", None)
2845
        max_memory = kwargs.pop("max_memory", None)
2846
        offload_folder = kwargs.pop("offload_folder", None)
2847
2848
        offload_state_dict = kwargs.pop("offload_state_dict", False)
        load_in_8bit = kwargs.pop("load_in_8bit", False)
2849
        load_in_4bit = kwargs.pop("load_in_4bit", False)
2850
        quantization_config = kwargs.pop("quantization_config", None)
2851
        subfolder = kwargs.pop("subfolder", "")
2852
        commit_hash = kwargs.pop("_commit_hash", None)
2853
        variant = kwargs.pop("variant", None)
2854
        adapter_kwargs = kwargs.pop("adapter_kwargs", {})
2855
        adapter_name = kwargs.pop("adapter_name", "default")
2856
        use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False)
2857

2858
2859
2860
        if is_fsdp_enabled():
            low_cpu_mem_usage = True

2861
2862
        if use_auth_token is not None:
            warnings.warn(
2863
2864
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
2865
2866
2867
2868
2869
2870
2871
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

2872
        if token is not None and adapter_kwargs is not None and "token" not in adapter_kwargs:
2873
2874
            adapter_kwargs["token"] = token

2875
2876
        if use_safetensors is None and not is_safetensors_available():
            use_safetensors = False
2877

2878
        if is_bitsandbytes_available():
2879
            is_4bit_serializable = version.parse(importlib.metadata.version("bitsandbytes")) >= version.parse("0.41.3")
2880
            is_8bit_serializable = version.parse(importlib.metadata.version("bitsandbytes")) > version.parse("0.37.2")
2881
        else:
2882
            is_4bit_serializable = False
2883
2884
            is_8bit_serializable = False

2885
2886
2887
2888
2889
        if trust_remote_code is True:
            logger.warning(
                "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is"
                " ignored."
            )
2890

2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
        if commit_hash is None:
            if not isinstance(config, PretrainedConfig):
                # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible
                resolved_config_file = cached_file(
                    pretrained_model_name_or_path,
                    CONFIG_NAME,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
                    token=token,
                    revision=revision,
                    subfolder=subfolder,
                    _raise_exceptions_for_missing_entries=False,
                    _raise_exceptions_for_connection_errors=False,
                )
                commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
            else:
                commit_hash = getattr(config, "_commit_hash", None)

2912
        if is_peft_available():
2913
2914
            _adapter_model_path = adapter_kwargs.pop("_adapter_model_path", None)

2915
2916
2917
2918
2919
2920
2921
2922
2923
            if _adapter_model_path is None:
                _adapter_model_path = find_adapter_config_file(
                    pretrained_model_name_or_path,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
                    _commit_hash=commit_hash,
2924
                    **adapter_kwargs,
2925
2926
                )
            if _adapter_model_path is not None and os.path.isfile(_adapter_model_path):
2927
                with open(_adapter_model_path, "r", encoding="utf-8") as f:
2928
                    _adapter_model_path = pretrained_model_name_or_path
2929
                    pretrained_model_name_or_path = json.load(f)["base_model_name_or_path"]
2930
2931
        else:
            _adapter_model_path = None
2932

2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
        # change device_map into a map if we passed an int, a str or a torch.device
        if isinstance(device_map, torch.device):
            device_map = {"": device_map}
        elif isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
            try:
                device_map = {"": torch.device(device_map)}
            except RuntimeError:
                raise ValueError(
                    "When passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or "
                    f"'auto', 'balanced', 'balanced_low_0', 'sequential' but found {device_map}."
                )
        elif isinstance(device_map, int):
            if device_map < 0:
                raise ValueError(
                    "You can't pass device_map as a negative int. If you want to put the model on the cpu, pass device_map = 'cpu' "
                )
            else:
                device_map = {"": device_map}

2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
        if device_map is not None:
            if low_cpu_mem_usage is None:
                low_cpu_mem_usage = True
            elif not low_cpu_mem_usage:
                raise ValueError("Passing along a `device_map` requires `low_cpu_mem_usage=True`")

        if low_cpu_mem_usage:
            if is_deepspeed_zero3_enabled():
                raise ValueError(
                    "DeepSpeed Zero-3 is not compatible with `low_cpu_mem_usage=True` or with passing a `device_map`."
                )
            elif not is_accelerate_available():
                raise ImportError(
                    "Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`"
                )
2967

Marc Sun's avatar
Marc Sun committed
2968
        quantization_method_from_args = None
2969

Marc Sun's avatar
Marc Sun committed
2970
2971
2972
2973
2974
2975
2976
        if quantization_config is not None:
            quantization_method_from_args = getattr(
                quantization_config, "quant_method", QuantizationMethod.BITS_AND_BYTES
            )

        if quantization_config is None and (load_in_8bit or load_in_4bit):
            quantization_method_from_args = QuantizationMethod.BITS_AND_BYTES
2977
            quantization_config, kwargs = BitsAndBytesConfig.from_dict(
2978
2979
2980
                config_dict={"load_in_8bit": load_in_8bit, "load_in_4bit": load_in_4bit},
                return_unused_kwargs=True,
                **kwargs,
2981
            )
Marc Sun's avatar
Marc Sun committed
2982
        elif quantization_method_from_args == QuantizationMethod.BITS_AND_BYTES:
2983
            load_in_8bit = quantization_config.load_in_8bit
2984
            load_in_4bit = quantization_config.load_in_4bit
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995

            quantization_config_kwargs = {
                k: v for k, v in kwargs.items() if k in inspect.signature(BitsAndBytesConfig).parameters
            }

            if len(quantization_config_kwargs) > 0:
                raise ValueError(
                    "You can't pass `load_in_8bit` or any other `BitsAndBytesConfig` argument as a kwarg when passing "
                    "`quantization_config` argument at the same time."
                )

2996
        if load_in_8bit or load_in_4bit:
2997
2998
            if not torch.cuda.is_available():
                raise RuntimeError("No GPU found. A GPU is needed for quantization.")
2999
3000
3001
3002
            if not (is_accelerate_available() and is_bitsandbytes_available()):
                raise ImportError(
                    "Using `load_in_8bit=True` requires Accelerate: `pip install accelerate` and the latest version of"
                    " bitsandbytes `pip install -i https://test.pypi.org/simple/ bitsandbytes` or"
3003
                    " `pip install bitsandbytes`."
3004
                )
3005
3006

            if torch_dtype is None:
3007
                # We force the `dtype` to be float16, this is a requirement from `bitsandbytes`
3008
                logger.info(
3009
                    f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to "
3010
                    "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. "
3011
3012
                    "Pass your own torch_dtype to specify the dtype of the remaining non-linear layers or pass"
                    " torch_dtype=torch.float16 to remove this warning."
3013
                )
3014
                torch_dtype = torch.float16
3015

3016
            if device_map is None:
3017
                device_map = {"": torch.cuda.current_device()}
3018
                logger.info(
3019
3020
                    "The device_map was not initialized. "
                    "Setting device_map to {'':torch.cuda.current_device()}. "
3021
                    "If you want to use the model for inference, please set device_map ='auto' "
3022
                )
3023
3024
3025
                if low_cpu_mem_usage is None:
                    low_cpu_mem_usage = True

3026
3027
            if from_tf or from_flax:
                raise ValueError(
3028
                    "Converting into 4-bit or 8-bit weights from tf/flax weights is currently not supported, please make"
3029
3030
3031
                    " sure the weights are in PyTorch format."
                )

3032
3033
3034
        user_agent = {"file_type": "model", "framework": "pytorch", "from_auto_class": from_auto_class}
        if from_pipeline is not None:
            user_agent["using_pipeline"] = from_pipeline
thomwolf's avatar
thomwolf committed
3035

3036
3037
3038
3039
        if is_offline_mode() and not local_files_only:
            logger.info("Offline mode: forcing local_files_only=True")
            local_files_only = True

3040
3041
3042
        # Load config if we don't provide a configuration
        if not isinstance(config, PretrainedConfig):
            config_path = config if config is not None else pretrained_model_name_or_path
3043
            config, model_kwargs = cls.config_class.from_pretrained(
3044
3045
3046
                config_path,
                cache_dir=cache_dir,
                return_unused_kwargs=True,
3047
                force_download=force_download,
3048
                resume_download=resume_download,
3049
                proxies=proxies,
3050
                local_files_only=local_files_only,
3051
                token=token,
Julien Chaumond's avatar
Julien Chaumond committed
3052
                revision=revision,
3053
                subfolder=subfolder,
3054
3055
                _from_auto=from_auto_class,
                _from_pipeline=from_pipeline,
3056
                **kwargs,
3057
3058
            )
        else:
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
            # In case one passes a config to `from_pretrained` + "attn_implementation"
            # override the `_attn_implementation` attribute to `attn_implementation` of the kwargs
            # Please see: https://github.com/huggingface/transformers/issues/28038

            # Overwrite `config._attn_implementation` by the one from the kwargs --> in auto-factory
            # we pop attn_implementation from the kwargs but this handles the case where users
            # passes manually the config to `from_pretrained`.
            config = copy.deepcopy(config)

            kwarg_attn_imp = kwargs.pop("attn_implementation", None)
            if kwarg_attn_imp is not None and config._attn_implementation != kwarg_attn_imp:
                config._attn_implementation = kwarg_attn_imp
3071
            model_kwargs = kwargs
3072

Marc Sun's avatar
Marc Sun committed
3073
3074
3075
3076
3077
3078
        quantizer = None
        quantization_method_from_config = None
        if hasattr(config, "quantization_config"):
            quantization_method_from_config = config.quantization_config.get(
                "quant_method", QuantizationMethod.BITS_AND_BYTES
            )
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089

        if (
            quantization_method_from_args is not None
            and quantization_method_from_args == QuantizationMethod.AWQ
            and quantization_method_from_config is None
        ):
            raise ValueError(
                "You cannot quantize with AWQ a non-quantized model using transformers, please refer to the quantization documentation"
                " to read more about how to quantize models with AWQ algorithm https://huggingface.co/docs/transformers/main_classes/quantization"
            )

3090
3091
3092
3093
3094
3095
        if quantization_method_from_config is not None and quantization_method_from_args is not None:
            if quantization_method_from_config != quantization_method_from_args:
                raise ValueError(
                    f"The model is already quantized with {quantization_method_from_config}. "
                    f"You can't quantize it again with {quantization_method_from_args}"
                )
3096
3097
3098
3099
3100

        if (
            quantization_method_from_config in (QuantizationMethod.GPTQ, QuantizationMethod.AWQ)
            and quantization_method_from_args is not None
        ):
Marc Sun's avatar
Marc Sun committed
3101
3102
3103
3104
3105
            loading_attr_dict = quantization_config.get_loading_attributes()
            for attr, val in loading_attr_dict.items():
                config.quantization_config[attr] = val
            quantization_method_from_args = None
            logger.warning(
3106
3107
3108
                f"You passed `quantization_config` to `from_pretrained` but the model you're loading already has a "
                f"`quantization_config` attribute and has already quantized weights. However, loading attributes"
                f" (e.g. {list(loading_attr_dict.keys())}) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored."
Marc Sun's avatar
Marc Sun committed
3109
3110
3111
3112
3113
            )
        if (
            quantization_method_from_args == QuantizationMethod.GPTQ
            or quantization_method_from_config == QuantizationMethod.GPTQ
        ):
3114
3115
            gptq_supports_cpu = version.parse(importlib.metadata.version("auto-gptq")) > version.parse("0.4.2")
            if not gptq_supports_cpu and not torch.cuda.is_available():
Marc Sun's avatar
Marc Sun committed
3116
3117
3118
                raise RuntimeError("GPU is required to quantize or run quantize model.")
            elif not (is_optimum_available() and is_auto_gptq_available()):
                raise ImportError(
3119
3120
3121
3122
3123
                    "Loading a GPTQ quantized model requires optimum (`pip install optimum`) and auto-gptq library (`pip install auto-gptq`)"
                )
            elif version.parse(importlib.metadata.version("auto_gptq")) < version.parse("0.4.2"):
                raise ImportError(
                    "You need a version of auto_gptq >= 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq`"
Marc Sun's avatar
Marc Sun committed
3124
3125
3126
3127
3128
3129
3130
                )
            else:
                # Need to protect the import
                from optimum.gptq import GPTQQuantizer
            if quantization_method_from_config == QuantizationMethod.GPTQ:
                quantization_config = GPTQConfig.from_dict(config.quantization_config)
                config.quantization_config = quantization_config
3131
3132
3133
3134
            if torch_dtype is None:
                torch_dtype = torch.float16
            else:
                logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with GPTQ.")
Marc Sun's avatar
Marc Sun committed
3135
            quantizer = GPTQQuantizer.from_dict(quantization_config.to_dict_optimum())
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
        elif quantization_method_from_config == QuantizationMethod.AWQ:
            if not torch.cuda.is_available():
                raise RuntimeError("GPU is required to run AWQ quantized model.")

            if not is_auto_awq_available():
                raise ImportError("Loading an AWQ quantized model requires auto-awq library (`pip install autoawq`)")

            if not is_accelerate_available():
                raise ImportError("Loading an AWQ quantized model requires accelerate (`pip install accelerate`)")

            if device_map is None:
                logger.warning(
                    "You have loaded an AWQ model on CPU and have a CUDA device available, make sure to set "
                    "your model on a GPU device in order to run your model."
                )
            elif device_map is not None:
                if isinstance(device_map, dict) and ("cpu" in device_map.values() or "disk" in device_map.values()):
                    raise ValueError(
                        "You are attempting to load an AWQ model with a device_map that contains a CPU or disk device."
                        " This is not supported. Please remove the CPU or disk device from the device_map."
                    )

            if torch_dtype is None:
                torch_dtype = torch.float16
            else:
                logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AWQ.")

            # Force-set to `True` for more mem efficiency
            if low_cpu_mem_usage is None:
                low_cpu_mem_usage = True
Marc Sun's avatar
Marc Sun committed
3166

3167
3168
        if quantization_method_from_args == QuantizationMethod.BITS_AND_BYTES and (
            (is_8bit_serializable and load_in_8bit) or (is_4bit_serializable and load_in_4bit)
Marc Sun's avatar
Marc Sun committed
3169
3170
        ):
            if quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES:
3171
3172
3173
3174
3175
3176
                logger.warning(
                    "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a"
                    " `quantization_config` attribute. The `quantization_config` attribute will be overwritten with the"
                    " one you passed to `from_pretrained`."
                )
            config.quantization_config = quantization_config
Marc Sun's avatar
Marc Sun committed
3177
        elif (
3178
3179
            (is_8bit_serializable or is_4bit_serializable)
            and not (load_in_8bit or load_in_4bit)
Marc Sun's avatar
Marc Sun committed
3180
3181
            and quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES
        ):
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
            quantization_config = config.quantization_config
            if isinstance(quantization_config, dict):
                quantization_config = BitsAndBytesConfig.from_dict(quantization_config, return_unused_kwargs=False)
            elif isinstance(quantization_config, BitsAndBytesConfig):
                pass
            else:
                raise ValueError(
                    f"Invalid type for `quantization_config`: {type(quantization_config)}. Should be a `dict` or a"
                    " `BitsAndBytesConfig` instance."
                )

            load_in_8bit = quantization_config.load_in_8bit
3194
            load_in_4bit = quantization_config.load_in_4bit
3195

3196
            if load_in_8bit or load_in_4bit:
3197
3198
                if torch_dtype is None:
                    torch_dtype = torch.float16
3199
                if device_map is None:
3200
3201
3202
3203
3204
                    if torch.cuda.is_available():
                        device_map = {"": torch.cuda.current_device()}
                    else:
                        raise RuntimeError("No GPU found. A GPU is needed for quantization.")
                    logger.info(
3205
3206
                        "The device_map was not initialized. "
                        "Setting device_map to {'':torch.cuda.current_device()}. "
3207
3208
3209
3210
                        "If you want to use the model for inference, please set device_map ='auto' "
                    )
                    if low_cpu_mem_usage is None:
                        low_cpu_mem_usage = True
3211

Marc Sun's avatar
Marc Sun committed
3212
3213
        elif (
            not is_8bit_serializable
3214
            and not (load_in_8bit or load_in_4bit)
Marc Sun's avatar
Marc Sun committed
3215
3216
            and quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES
        ):
3217
3218
            logger.warning(
                "Detected the presence of a `quantization_config` attribute in the model's configuration but you don't have the correct"
3219
                " `bitsandbytes` version to support 4 and 8 bit serialization. Please install the latest version of `bitsandbytes` with "
3220
3221
3222
                " `pip install --upgrade bitsandbytes`."
            )

Sylvain Gugger's avatar
Sylvain Gugger committed
3223
3224
3225
3226
        # This variable will flag if we're loading a sharded checkpoint. In this case the archive file is just the
        # index of the files.
        is_sharded = False
        sharded_metadata = None
thomwolf's avatar
thomwolf committed
3227
        # Load model
Yih-Dar's avatar
Yih-Dar committed
3228
3229
        loading_info = None

3230
3231
3232
3233
        # Keep in fp32 modules
        keep_in_fp32_modules = None
        use_keep_in_fp32_modules = False

thomwolf's avatar
thomwolf committed
3234
        if pretrained_model_name_or_path is not None:
3235
            pretrained_model_name_or_path = str(pretrained_model_name_or_path)
3236
3237
            is_local = os.path.isdir(pretrained_model_name_or_path)
            if is_local:
3238
3239
3240
                if from_tf and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                ):
3241
                    # Load from a TF 1.0 checkpoint in priority if from_tf
3242
3243
3244
3245
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                elif from_tf and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)
                ):
3246
                    # Load from a TF 2.0 checkpoint in priority if from_tf
3247
3248
3249
3250
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)
                elif from_flax and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
                ):
3251
                    # Load from a Flax checkpoint in priority if from_flax
3252
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
3253
                elif use_safetensors is not False and os.path.isfile(
3254
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant))
3255
3256
                ):
                    # Load from a safetensors checkpoint
3257
3258
3259
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant)
                    )
3260
                elif use_safetensors is not False and os.path.isfile(
3261
3262
3263
                    os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)
                    )
3264
3265
                ):
                    # Load from a sharded safetensors checkpoint
3266
3267
3268
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)
                    )
3269
                    is_sharded = True
3270
3271
3272
                elif os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_NAME, variant))
                ):
thomwolf's avatar
thomwolf committed
3273
                    # Load from a PyTorch checkpoint
3274
3275
3276
3277
3278
3279
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_NAME, variant)
                    )
                elif os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_INDEX_NAME, variant))
                ):
Sylvain Gugger's avatar
Sylvain Gugger committed
3280
                    # Load from a sharded PyTorch checkpoint
3281
3282
3283
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_INDEX_NAME, variant)
                    )
Sylvain Gugger's avatar
Sylvain Gugger committed
3284
                    is_sharded = True
3285
3286
                # At this stage we don't have a weight file so we will raise an error.
                elif os.path.isfile(
3287
3288
                    os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                ) or os.path.isfile(os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)):
3289
                    raise EnvironmentError(
3290
3291
3292
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path} but there is a file for TensorFlow weights. Use"
                        " `from_tf=True` to load this model from those weights."
3293
                    )
3294
                elif os.path.isfile(os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)):
3295
                    raise EnvironmentError(
3296
3297
3298
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path} but there is a file for Flax weights. Use `from_flax=True`"
                        " to load this model from those weights."
3299
                    )
3300
3301
3302
3303
3304
                elif use_safetensors:
                    raise EnvironmentError(
                        f"Error no file named {_add_variant(SAFE_WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path}."
                    )
thomwolf's avatar
thomwolf committed
3305
                else:
3306
                    raise EnvironmentError(
3307
3308
3309
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)}, {TF2_WEIGHTS_NAME},"
                        f" {TF_WEIGHTS_NAME + '.index'} or {FLAX_WEIGHTS_NAME} found in directory"
                        f" {pretrained_model_name_or_path}."
3310
                    )
3311
            elif os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path)):
3312
                archive_file = pretrained_model_name_or_path
3313
                is_local = True
3314
            elif os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path + ".index")):
3315
3316
3317
3318
3319
                if not from_tf:
                    raise ValueError(
                        f"We found a TensorFlow checkpoint at {pretrained_model_name_or_path + '.index'}, please set "
                        "from_tf to True to load from this checkpoint."
                    )
3320
                archive_file = os.path.join(subfolder, pretrained_model_name_or_path + ".index")
3321
                is_local = True
3322
            elif is_remote_url(pretrained_model_name_or_path):
3323
                filename = pretrained_model_name_or_path
3324
                resolved_archive_file = download_url(pretrained_model_name_or_path)
3325
            else:
3326
3327
3328
3329
3330
                # set correct filename
                if from_tf:
                    filename = TF2_WEIGHTS_NAME
                elif from_flax:
                    filename = FLAX_WEIGHTS_NAME
3331
                elif use_safetensors is not False:
3332
                    filename = _add_variant(SAFE_WEIGHTS_NAME, variant)
3333
                else:
3334
                    filename = _add_variant(WEIGHTS_NAME, variant)
3335

3336
3337
                try:
                    # Load from URL or cache if already cached
3338
3339
3340
3341
3342
3343
                    cached_file_kwargs = {
                        "cache_dir": cache_dir,
                        "force_download": force_download,
                        "proxies": proxies,
                        "resume_download": resume_download,
                        "local_files_only": local_files_only,
3344
                        "token": token,
3345
3346
3347
3348
3349
3350
                        "user_agent": user_agent,
                        "revision": revision,
                        "subfolder": subfolder,
                        "_raise_exceptions_for_missing_entries": False,
                        "_commit_hash": commit_hash,
                    }
3351
                    resolved_archive_file = cached_file(pretrained_model_name_or_path, filename, **cached_file_kwargs)
3352

3353
                    # Since we set _raise_exceptions_for_missing_entries=False, we don't get an exception but a None
3354
                    # result when internet is up, the repo and revision exist, but the file does not.
3355
                    if resolved_archive_file is None and filename == _add_variant(SAFE_WEIGHTS_NAME, variant):
3356
3357
                        # Maybe the checkpoint is sharded, we try to grab the index name in this case.
                        resolved_archive_file = cached_file(
3358
3359
3360
                            pretrained_model_name_or_path,
                            _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant),
                            **cached_file_kwargs,
3361
3362
3363
                        )
                        if resolved_archive_file is not None:
                            is_sharded = True
3364
                        elif use_safetensors:
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
                            if revision == "main":
                                resolved_archive_file, revision, is_sharded = auto_conversion(
                                    pretrained_model_name_or_path, **cached_file_kwargs
                                )
                            cached_file_kwargs["revision"] = revision
                            if resolved_archive_file is None:
                                raise EnvironmentError(
                                    f"{pretrained_model_name_or_path} does not appear to have a file named"
                                    f" {_add_variant(SAFE_WEIGHTS_NAME, variant)} or {_add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)} "
                                    "and thus cannot be loaded with `safetensors`. Please make sure that the model has "
                                    "been saved with `safe_serialization=True` or do not set `use_safetensors=True`."
                                )
3377
3378
                        else:
                            # This repo has no safetensors file of any kind, we switch to PyTorch.
3379
                            filename = _add_variant(WEIGHTS_NAME, variant)
3380
                            resolved_archive_file = cached_file(
3381
                                pretrained_model_name_or_path, filename, **cached_file_kwargs
3382
                            )
3383
                    if resolved_archive_file is None and filename == _add_variant(WEIGHTS_NAME, variant):
Sylvain Gugger's avatar
Sylvain Gugger committed
3384
                        # Maybe the checkpoint is sharded, we try to grab the index name in this case.
3385
                        resolved_archive_file = cached_file(
3386
3387
3388
                            pretrained_model_name_or_path,
                            _add_variant(WEIGHTS_INDEX_NAME, variant),
                            **cached_file_kwargs,
3389
                        )
3390
3391
3392
                        if resolved_archive_file is not None:
                            is_sharded = True
                    if resolved_archive_file is None:
Sylvain Gugger's avatar
Sylvain Gugger committed
3393
3394
3395
3396
3397
                        # Otherwise, maybe there is a TF or Flax model file.  We try those to give a helpful error
                        # message.
                        has_file_kwargs = {
                            "revision": revision,
                            "proxies": proxies,
3398
                            "token": token,
Sylvain Gugger's avatar
Sylvain Gugger committed
3399
3400
3401
                        }
                        if has_file(pretrained_model_name_or_path, TF2_WEIGHTS_NAME, **has_file_kwargs):
                            raise EnvironmentError(
Sylvain Gugger's avatar
Sylvain Gugger committed
3402
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
3403
3404
                                f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file for TensorFlow weights."
                                " Use `from_tf=True` to load this model from those weights."
Sylvain Gugger's avatar
Sylvain Gugger committed
3405
3406
3407
                            )
                        elif has_file(pretrained_model_name_or_path, FLAX_WEIGHTS_NAME, **has_file_kwargs):
                            raise EnvironmentError(
Sylvain Gugger's avatar
Sylvain Gugger committed
3408
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
                                f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file for Flax weights. Use"
                                " `from_flax=True` to load this model from those weights."
                            )
                        elif variant is not None and has_file(
                            pretrained_model_name_or_path, WEIGHTS_NAME, **has_file_kwargs
                        ):
                            raise EnvironmentError(
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
                                f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file without the variant"
                                f" {variant}. Use `variant=None` to load this model from those weights."
Sylvain Gugger's avatar
Sylvain Gugger committed
3419
3420
3421
                            )
                        else:
                            raise EnvironmentError(
3422
3423
3424
                                f"{pretrained_model_name_or_path} does not appear to have a file named"
                                f" {_add_variant(WEIGHTS_NAME, variant)}, {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME} or"
                                f" {FLAX_WEIGHTS_NAME}."
Sylvain Gugger's avatar
Sylvain Gugger committed
3425
                            )
3426
3427
3428
3429
                except EnvironmentError:
                    # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted
                    # to the original exception.
                    raise
3430
                except Exception as e:
3431
                    # For any other exception, we throw a generic error.
3432
                    raise EnvironmentError(
3433
3434
3435
                        f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it"
                        " from 'https://huggingface.co/models', make sure you don't have a local directory with the"
                        f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
3436
3437
                        f" directory containing a file named {_add_variant(WEIGHTS_NAME, variant)},"
                        f" {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME} or {FLAX_WEIGHTS_NAME}."
3438
                    ) from e
3439

3440
            if is_local:
3441
                logger.info(f"loading weights file {archive_file}")
3442
                resolved_archive_file = archive_file
3443
            else:
3444
                logger.info(f"loading weights file {filename} from cache at {resolved_archive_file}")
3445
        else:
thomwolf's avatar
thomwolf committed
3446
            resolved_archive_file = None
3447

Sylvain Gugger's avatar
Sylvain Gugger committed
3448
3449
        # We'll need to download and cache each checkpoint shard if the checkpoint is sharded.
        if is_sharded:
3450
            # rsolved_archive_file becomes a list of files that point to the different checkpoint shards in this case.
Sylvain Gugger's avatar
Sylvain Gugger committed
3451
3452
3453
3454
3455
3456
3457
3458
            resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
                pretrained_model_name_or_path,
                resolved_archive_file,
                cache_dir=cache_dir,
                force_download=force_download,
                proxies=proxies,
                resume_download=resume_download,
                local_files_only=local_files_only,
3459
                token=token,
Sylvain Gugger's avatar
Sylvain Gugger committed
3460
3461
                user_agent=user_agent,
                revision=revision,
3462
                subfolder=subfolder,
3463
                _commit_hash=commit_hash,
Sylvain Gugger's avatar
Sylvain Gugger committed
3464
3465
            )

3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
        if (
            is_safetensors_available()
            and isinstance(resolved_archive_file, str)
            and resolved_archive_file.endswith(".safetensors")
        ):
            with safe_open(resolved_archive_file, framework="pt") as f:
                metadata = f.metadata()

            if metadata.get("format") == "pt":
                pass
            elif metadata.get("format") == "tf":
                from_tf = True
                logger.info("A TensorFlow safetensors file is being loaded in a PyTorch model.")
            elif metadata.get("format") == "flax":
                from_flax = True
                logger.info("A Flax safetensors file is being loaded in a PyTorch model.")
            else:
                raise ValueError(
                    f"Incompatible safetensors file. File metadata is not ['pt', 'tf', 'flax'] but {metadata.get('format')}"
                )

        from_pt = not (from_tf | from_flax)

3489
3490
        # load pt weights early so that we know which dtype to init the model under
        if from_pt:
3491
            if not is_sharded and state_dict is None:
Sylvain Gugger's avatar
Sylvain Gugger committed
3492
3493
                # Time to load the checkpoint
                state_dict = load_state_dict(resolved_archive_file)
3494

3495
3496
3497
            # set dtype to instantiate the model under:
            # 1. If torch_dtype is not None, we use that dtype
            # 2. If torch_dtype is "auto", we auto-detect dtype from the loaded state_dict, by checking its first
3498
            #    weights entry that is of a floating type - we assume all floating dtype weights are of the same dtype
3499
3500
            # we also may have config.torch_dtype available, but we won't rely on it till v5
            dtype_orig = None
3501

3502
3503
3504
            if torch_dtype is not None:
                if isinstance(torch_dtype, str):
                    if torch_dtype == "auto":
3505
3506
3507
                        if hasattr(config, "torch_dtype") and config.torch_dtype is not None:
                            torch_dtype = config.torch_dtype
                            logger.info(f"Will use torch_dtype={torch_dtype} as defined in model's config object")
Sylvain Gugger's avatar
Sylvain Gugger committed
3508
                        else:
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
                            if is_sharded and "dtype" in sharded_metadata:
                                torch_dtype = sharded_metadata["dtype"]
                            elif not is_sharded:
                                torch_dtype = get_state_dict_dtype(state_dict)
                            else:
                                one_state_dict = load_state_dict(resolved_archive_file[0])
                                torch_dtype = get_state_dict_dtype(one_state_dict)
                                del one_state_dict  # free CPU memory
                            logger.info(
                                "Since the `torch_dtype` attribute can't be found in model's config object, "
                                "will use torch_dtype={torch_dtype} as derived from model's weights"
                            )
3521
3522
                    else:
                        raise ValueError(
3523
                            f'`torch_dtype` can be either `torch.dtype` or `"auto"`, but received {torch_dtype}'
3524
3525
3526
                        )
                dtype_orig = cls._set_default_torch_dtype(torch_dtype)

3527
            # Check if `_keep_in_fp32_modules` is not None
3528
3529
            use_keep_in_fp32_modules = (cls._keep_in_fp32_modules is not None) and (
                torch_dtype == torch.float16 or load_in_4bit or load_in_8bit
3530
3531
            )

3532
3533
3534
            if is_sharded:
                loaded_state_dict_keys = sharded_metadata["all_checkpoint_keys"]
            else:
3535
                loaded_state_dict_keys = list(state_dict.keys())
3536
3537
3538
3539
            if low_cpu_mem_usage or (use_keep_in_fp32_modules and is_accelerate_available()):
                # In case some weights need to be kept in float32 and accelerate is not installed,
                # we later on want to take the path where state_dict is not None, that is the one
                # that do not require accelerate.
3540
                state_dict = None
3541

3542
3543
        config.name_or_path = pretrained_model_name_or_path

3544
        # Instantiate model.
3545
3546
        init_contexts = [no_init_weights(_enable=_fast_init)]

3547
3548
3549
3550
        if is_deepspeed_zero3_enabled():
            import deepspeed

            logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
3551
            init_contexts = [deepspeed.zero.Init(config_dict_or_path=deepspeed_config())] + init_contexts
3552
        elif load_in_8bit or load_in_4bit or low_cpu_mem_usage:
3553
3554
            init_contexts.append(init_empty_weights())

3555
3556
3557
3558
        config = copy.deepcopy(config)  # We do not want to modify the config inplace in from_pretrained.
        config = cls._autoset_attn_implementation(
            config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, device_map=device_map
        )
3559

3560
        with ContextManagers(init_contexts):
3561
            # Let's make sure we don't run the init function of buffer modules
3562
3563
            model = cls(config, *model_args, **model_kwargs)

3564
3565
3566
        # make sure we use the model's config since the __init__ call might have copied it
        config = model.config

3567
3568
        # Check first if we are `from_pt`
        if use_keep_in_fp32_modules:
3569
            if is_accelerate_available() and not is_deepspeed_zero3_enabled():
3570
                low_cpu_mem_usage = True
3571
3572
3573
3574
            keep_in_fp32_modules = model._keep_in_fp32_modules
        else:
            keep_in_fp32_modules = []

3575
        if load_in_8bit or load_in_4bit:
3576
            from .integrations import get_keys_to_not_convert, replace_with_bnb_linear
3577

3578
            llm_int8_skip_modules = quantization_config.llm_int8_skip_modules
3579
            load_in_8bit_fp32_cpu_offload = quantization_config.llm_int8_enable_fp32_cpu_offload
3580
3581
3582
3583
            if load_in_8bit:
                logger.info("Detected 8-bit loading: activating 8-bit loading for this model")
            else:
                logger.info("Detected 4-bit loading: activating 4-bit loading for this model")
3584

3585
            # We keep some modules such as the lm_head in their original dtype for numerical stability reasons
3586
            if llm_int8_skip_modules is None:
3587
3588
                modules_to_not_convert = get_keys_to_not_convert(model)
            else:
3589
                modules_to_not_convert = llm_int8_skip_modules
3590
3591
3592
3593
3594
3595

            if not isinstance(modules_to_not_convert, list):
                modules_to_not_convert = [modules_to_not_convert]

            modules_to_not_convert.extend(keep_in_fp32_modules)

3596
3597
3598
3599
3600
3601
3602
            # Extend the modules to not convert to keys that are supposed to be offloaded to `cpu` or `disk`
            if isinstance(device_map, dict) and len(device_map.keys()) > 1:
                keys_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]]

                if len(keys_on_cpu) > 0 and not load_in_8bit_fp32_cpu_offload:
                    raise ValueError(
                        "If you want to offload some keys to `cpu` or `disk`, you need to set "
Younes Belkada's avatar
Younes Belkada committed
3603
                        "`llm_int8_enable_fp32_cpu_offload=True`. Note that these modules will not be "
3604
3605
3606
3607
3608
                        " converted to 8-bit but kept in 32-bit."
                    )

                modules_to_not_convert.extend(keys_on_cpu)

3609
            supports_4bit = version.parse(importlib.metadata.version("bitsandbytes")) >= version.parse("0.39.0")
3610
3611
3612
3613
3614
3615
3616
3617
3618

            if load_in_4bit and not supports_4bit:
                raise ValueError(
                    "You have a version of `bitsandbytes` that is not compatible with 4bit inference and training"
                    " make sure you have the latest version of `bitsandbytes` installed"
                )

            model = replace_with_bnb_linear(
                model, modules_to_not_convert=modules_to_not_convert, quantization_config=quantization_config
3619
            )
3620
            # training in 8-bit is only available in 0.37.0+
3621
            model._is_quantized_training_enabled = version.parse(
3622
                importlib.metadata.version("bitsandbytes")
3623
            ) >= version.parse("0.37.0")
3624

3625
            config.quantization_config = quantization_config
3626
            model.is_8bit_serializable = is_8bit_serializable
3627
            model.is_4bit_serializable = is_4bit_serializable
3628

3629
3630
        if load_in_8bit and torch_dtype is None:
            logger.warning(
3631
                "You are loading your model in 8bit but you did not specify a `torch_dtype` attribute. "
3632
3633
                "All non-linear modules will be loaded in full precision."
                " If you want to load the other modules in other precision, please specify a `torch_dtype` attribute."
3634
            )
Marc Sun's avatar
Marc Sun committed
3635
3636
3637
        if quantization_method_from_config == QuantizationMethod.GPTQ:
            model = quantizer.convert_model(model)
            model._is_quantized_training_enabled = True
3638
        elif quantization_method_from_config == QuantizationMethod.AWQ:
3639
            from .integrations import fuse_awq_modules, get_keys_to_not_convert, replace_with_awq_linear
3640
3641
3642
3643
3644

            modules_to_not_convert = get_keys_to_not_convert(model)

            if quantization_config is None:
                quantization_config = AwqConfig.from_dict(config.quantization_config)
3645
3646
3647
3648
            # In case a user passes a `AwqConfig` with `do_fuse=True` for models that have
            # a `modules_to_not_convert` attribute we need to manually set that attribute into the
            # passed `quantization_config`
            elif (
3649
                getattr(quantization_config, "modules_to_not_convert", None) is None
3650
3651
3652
                and "modules_to_not_convert" in config.quantization_config
            ):
                quantization_config.modules_to_not_convert = config.quantization_config["modules_to_not_convert"]
3653

3654
3655
3656
            if quantization_config.modules_to_not_convert is not None:
                modules_to_not_convert.extend(quantization_config.modules_to_not_convert)

3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
            model, has_been_replaced = replace_with_awq_linear(
                model, quantization_config=quantization_config, modules_to_not_convert=modules_to_not_convert
            )
            model._is_quantized_training_enabled = False

            if not has_been_replaced:
                logger.warning(
                    "You are loading an AWQ model but no linear modules were found in your model."
                    " Please double check your model architecture, or submit an issue on github if you think this is"
                    " a bug."
                )
Marc Sun's avatar
Marc Sun committed
3668
3669
3670
3671
3672
3673
3674

        if quantization_method_from_config is not None:
            model.quantization_method = quantization_method_from_config
        elif quantization_method_from_args is not None:
            model.quantization_method = quantization_method_from_args
        if hasattr(model, "quantization_method"):
            model.is_quantized = True
3675

3676
3677
3678
3679
3680
3681
            # We store the original dtype for quantized models as we cannot easily retrieve it
            # once the weights have been quantized
            # Note that once you have loaded a quantized model, you can't change its dtype so this will
            # remain a single source of truth
            config._pre_quantization_dtype = torch_dtype

3682
        if isinstance(device_map, str):
3683
            special_dtypes = {}
3684
            if load_in_8bit or load_in_4bit:
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
                special_dtypes.update(
                    {
                        name: torch_dtype
                        for name, _ in model.named_parameters()
                        if any(m in name for m in modules_to_not_convert)
                    }
                )

            special_dtypes.update(
                {
                    name: torch.float32
                    for name, _ in model.named_parameters()
                    if any(m in name for m in keep_in_fp32_modules)
                }
            )

3701
3702
3703
            target_dtype = torch_dtype

            if load_in_4bit:
3704
                if version.parse(importlib.metadata.version("accelerate")) > version.parse("0.19.0"):
3705
3706
3707
3708
3709
3710
                    from accelerate.utils import CustomDtype

                    target_dtype = CustomDtype.INT4
                else:
                    raise ValueError(
                        "You are using `device_map='auto'` on a 4bit loaded version of the model. To automatically compute"
3711
3712
                        " the appropriate device map, you should upgrade your `accelerate` library, "
                        "`pip install --upgrade accelerate` or install it from source to support fp4 auto device map "
3713
3714
3715
3716
3717
                        "calculation. You may encounter unexpected behavior, or pass your own device map"
                    )
            elif load_in_8bit:
                target_dtype = torch.int8

Marc Sun's avatar
Marc Sun committed
3718
            no_split_modules = model._get_no_split_modules(device_map)
3719
3720
3721
3722
3723
            if device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
                raise ValueError(
                    "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or "
                    "'sequential'."
                )
3724

3725
            device_map_kwargs = {"no_split_module_classes": no_split_modules}
3726
            if "special_dtypes" in inspect.signature(infer_auto_device_map).parameters:
3727
                device_map_kwargs["special_dtypes"] = special_dtypes
3728
            elif len(special_dtypes) > 0:
3729
                logger.warning(
3730
3731
3732
                    "This model has some weights that should be kept in higher precision, you need to upgrade "
                    "`accelerate` to properly deal with them (`pip install --upgrade accelerate`)."
                )
3733
            if device_map != "sequential":
3734
3735
                max_memory = get_balanced_memory(
                    model,
3736
                    dtype=target_dtype,
3737
                    low_zero=(device_map == "balanced_low_0"),
3738
                    max_memory=max_memory,
3739
                    **device_map_kwargs,
3740
                )
Marc Sun's avatar
Marc Sun committed
3741
3742
3743
3744
3745
            else:
                max_memory = get_max_memory(max_memory)
            if getattr(model, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
                # need more space for buffers that are created during quantization
                max_memory = {key: val * 0.90 for key, val in max_memory.items()}
3746
            device_map_kwargs["max_memory"] = max_memory
Marc Sun's avatar
Marc Sun committed
3747

3748
3749
            # Make sure tied weights are tied before creating the device map.
            model.tie_weights()
3750
            device_map = infer_auto_device_map(model, dtype=target_dtype, **device_map_kwargs)
3751

3752
            if load_in_8bit or load_in_4bit:
3753
                # The LM head / tied weights or any last module can stay on disk / CPU
3754
                device_map_without_lm_head = {
3755
                    key: device_map[key] for key in device_map.keys() if key not in modules_to_not_convert
3756
3757
                }
                if "cpu" in device_map_without_lm_head.values() or "disk" in device_map_without_lm_head.values():
3758
3759
3760
                    raise ValueError(
                        """
                        Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit
3761
3762
3763
3764
3765
                        the quantized model. If you want to dispatch the model on the CPU or the disk while keeping
                        these modules in 32-bit, you need to set `load_in_8bit_fp32_cpu_offload=True` and pass a custom
                        `device_map` to `from_pretrained`. Check
                        https://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-cpu-and-gpu
                        for more details.
3766
3767
                        """
                    )
3768
3769
                del device_map_without_lm_head

3770
3771
3772
3773
        elif device_map is not None:
            model.tie_weights()
            tied_params = find_tied_parameters(model)
            # check if we don't have tied param in different devices
3774
            check_tied_parameters_on_same_device(tied_params, device_map)
3775

3776
        if from_tf:
3777
            if resolved_archive_file.endswith(".index"):
3778
3779
3780
3781
3782
                # Load from a TensorFlow 1.X checkpoint - provided by original authors
                model = cls.load_tf_weights(model, config, resolved_archive_file[:-6])  # Remove the '.index'
            else:
                # Load from our TensorFlow 2.0 checkpoints
                try:
3783
                    from .modeling_tf_pytorch_utils import load_tf2_checkpoint_in_pytorch_model
3784

Yih-Dar's avatar
Yih-Dar committed
3785
3786
3787
                    model, loading_info = load_tf2_checkpoint_in_pytorch_model(
                        model, resolved_archive_file, allow_missing_keys=True, output_loading_info=True
                    )
3788
                except ImportError:
3789
                    logger.error(
Sylvain Gugger's avatar
Sylvain Gugger committed
3790
3791
3792
                        "Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed."
                        " Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation"
                        " instructions."
3793
                    )
3794
                    raise
3795
3796
3797
3798
3799
3800
3801
        elif from_flax:
            try:
                from .modeling_flax_pytorch_utils import load_flax_checkpoint_in_pytorch_model

                model = load_flax_checkpoint_in_pytorch_model(model, resolved_archive_file)
            except ImportError:
                logger.error(
Sylvain Gugger's avatar
Sylvain Gugger committed
3802
3803
3804
                    "Loading a Flax model in PyTorch, requires both PyTorch and Flax to be installed. Please see"
                    " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for"
                    " installation instructions."
3805
3806
                )
                raise
3807
        elif from_pt:
3808
3809
3810
            # restore default dtype
            if dtype_orig is not None:
                torch.set_default_dtype(dtype_orig)
Sylvain Gugger's avatar
Sylvain Gugger committed
3811
3812
3813
3814
3815
3816
3817
3818
            (
                model,
                missing_keys,
                unexpected_keys,
                mismatched_keys,
                offload_index,
                error_msgs,
            ) = cls._load_pretrained_model(
3819
3820
3821
3822
3823
3824
3825
3826
3827
                model,
                state_dict,
                loaded_state_dict_keys,  # XXX: rename?
                resolved_archive_file,
                pretrained_model_name_or_path,
                ignore_mismatched_sizes=ignore_mismatched_sizes,
                sharded_metadata=sharded_metadata,
                _fast_init=_fast_init,
                low_cpu_mem_usage=low_cpu_mem_usage,
3828
3829
3830
3831
                device_map=device_map,
                offload_folder=offload_folder,
                offload_state_dict=offload_state_dict,
                dtype=torch_dtype,
Marc Sun's avatar
Marc Sun committed
3832
                is_quantized=(getattr(model, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES),
3833
                keep_in_fp32_modules=keep_in_fp32_modules,
3834
            )
3835

3836
        model.is_loaded_in_4bit = load_in_4bit
Younes Belkada's avatar
Younes Belkada committed
3837
        model.is_loaded_in_8bit = load_in_8bit
3838

3839
3840
        # make sure token embedding weights are still tied if needed
        model.tie_weights()
3841

3842
        # Set model in evaluation mode to deactivate DropOut modules by default
3843
3844
        model.eval()

3845
        # If it is a model with generation capabilities, attempt to load the generation config
3846
        if model.can_generate() and pretrained_model_name_or_path is not None:
3847
3848
3849
3850
3851
3852
3853
3854
            try:
                model.generation_config = GenerationConfig.from_pretrained(
                    pretrained_model_name_or_path,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
3855
                    token=token,
3856
3857
3858
3859
3860
3861
                    revision=revision,
                    subfolder=subfolder,
                    _from_auto=from_auto_class,
                    _from_pipeline=from_pipeline,
                    **kwargs,
                )
3862
            except OSError:
3863
3864
3865
3866
3867
                logger.info(
                    "Generation config file not found, using a generation config created from the model config."
                )
                pass

3868
3869
3870
3871
3872
3873
3874
3875
        if (
            quantization_config is not None
            and quantization_config.quant_method == QuantizationMethod.AWQ
            and quantization_config.do_fuse
        ):
            model = fuse_awq_modules(model, config.quantization_config)
            model._awq_is_fused = True

3876
3877
        # Dispatch model with hooks on all devices if necessary
        if device_map is not None:
3878
3879
3880
3881
3882
            device_map_kwargs = {
                "device_map": device_map,
                "offload_dir": offload_folder,
                "offload_index": offload_index,
            }
3883
            if "skip_keys" in inspect.signature(dispatch_model).parameters:
3884
3885
                device_map_kwargs["skip_keys"] = model._skip_keys_device_placement
            dispatch_model(model, **device_map_kwargs)
3886

Marc Sun's avatar
Marc Sun committed
3887
3888
3889
3890
3891
3892
        if quantization_method_from_args == QuantizationMethod.GPTQ:
            if quantization_config.tokenizer is None:
                quantization_config.tokenizer = pretrained_model_name_or_path
            if cls.main_input_name != "input_ids":
                raise RuntimeError("We can only quantize pure text model.")
            quantizer.quantize_model(model, quantization_config.tokenizer)
Marc Sun's avatar
Marc Sun committed
3893
            config.quantization_config = GPTQConfig.from_dict_optimum(quantizer.to_dict())
Marc Sun's avatar
Marc Sun committed
3894
3895
3896
3897
            model._is_quantized_training_enabled = True
        if quantization_method_from_config == QuantizationMethod.GPTQ:
            model = quantizer.post_init_model(model)

3898
        if _adapter_model_path is not None:
3899
            model.load_adapter(
3900
                _adapter_model_path,
3901
3902
                adapter_name=adapter_name,
                token=token,
3903
                adapter_kwargs=adapter_kwargs,
3904
3905
            )

thomwolf's avatar
thomwolf committed
3906
        if output_loading_info:
Yih-Dar's avatar
Yih-Dar committed
3907
3908
3909
3910
3911
3912
3913
            if loading_info is None:
                loading_info = {
                    "missing_keys": missing_keys,
                    "unexpected_keys": unexpected_keys,
                    "mismatched_keys": mismatched_keys,
                    "error_msgs": error_msgs,
                }
thomwolf's avatar
thomwolf committed
3914
3915
            return model, loading_info

3916
3917
        return model

3918
    @classmethod
Sylvain Gugger's avatar
Sylvain Gugger committed
3919
3920
3921
3922
    def _load_pretrained_model(
        cls,
        model,
        state_dict,
3923
        loaded_keys,
Sylvain Gugger's avatar
Sylvain Gugger committed
3924
3925
3926
3927
3928
        resolved_archive_file,
        pretrained_model_name_or_path,
        ignore_mismatched_sizes=False,
        sharded_metadata=None,
        _fast_init=True,
3929
        low_cpu_mem_usage=False,
3930
3931
        device_map=None,
        offload_folder=None,
3932
        offload_state_dict=None,
3933
        dtype=None,
3934
        is_quantized=False,
3935
        keep_in_fp32_modules=None,
3936
    ):
Sylvain Gugger's avatar
Sylvain Gugger committed
3937
        is_safetensors = False
3938
        if is_quantized:
3939
            from .integrations import set_module_quantized_tensor_to_device
3940

Sylvain Gugger's avatar
Sylvain Gugger committed
3941
        if device_map is not None and "disk" in device_map.values():
Sylvain Gugger's avatar
Sylvain Gugger committed
3942
3943
3944
3945
3946
            archive_file = (
                resolved_archive_file[0] if isinstance(resolved_archive_file, (list, tuple)) else resolved_archive_file
            )
            is_safetensors = archive_file.endswith(".safetensors")
            if offload_folder is None and not is_safetensors:
Sylvain Gugger's avatar
Sylvain Gugger committed
3947
3948
                raise ValueError(
                    "The current `device_map` had weights offloaded to the disk. Please provide an `offload_folder`"
Sylvain Gugger's avatar
Sylvain Gugger committed
3949
3950
                    " for them. Alternatively, make sure you have `safetensors` installed if the model you are using"
                    " offers the weights in this format."
Sylvain Gugger's avatar
Sylvain Gugger committed
3951
                )
Sylvain Gugger's avatar
Sylvain Gugger committed
3952
3953
            if offload_folder is not None:
                os.makedirs(offload_folder, exist_ok=True)
3954
3955
3956
            if offload_state_dict is None:
                offload_state_dict = True

3957
        is_sharded_safetensors = is_safetensors and sharded_metadata is not None
Patrick von Platen's avatar
Patrick von Platen committed
3958
3959
3960
3961

        # tie the model weights before retrieving the state_dict
        model.tie_weights()

3962
        # Retrieve missing & unexpected_keys
3963
3964
        model_state_dict = model.state_dict()
        expected_keys = list(model_state_dict.keys())
3965
3966
        prefix = model.base_model_prefix

Sylvain Gugger's avatar
Sylvain Gugger committed
3967
3968
3969
3970
3971
3972
3973
        def _fix_key(key):
            if "beta" in key:
                return key.replace("beta", "bias")
            if "gamma" in key:
                return key.replace("gamma", "weight")
            return key

3974
        original_loaded_keys = loaded_keys
Sylvain Gugger's avatar
Sylvain Gugger committed
3975
3976
        loaded_keys = [_fix_key(key) for key in loaded_keys]

3977
3978
3979
3980
3981
3982
        if len(prefix) > 0:
            has_prefix_module = any(s.startswith(prefix) for s in loaded_keys)
            expects_prefix_module = any(s.startswith(prefix) for s in expected_keys)
        else:
            has_prefix_module = False
            expects_prefix_module = False
Patrick von Platen's avatar
Patrick von Platen committed
3983
3984
3985

        # key re-naming operations are never done on the keys
        # that are loaded, but always on the keys of the newly initialized model
3986
3987
        remove_prefix_from_model = not has_prefix_module and expects_prefix_module
        add_prefix_to_model = has_prefix_module and not expects_prefix_module
3988

3989
        if remove_prefix_from_model:
3990
3991
3992
            _prefix = f"{prefix}."
            expected_keys_not_prefixed = [s for s in expected_keys if not s.startswith(_prefix)]
            expected_keys = [s[len(_prefix) :] if s.startswith(_prefix) else s for s in expected_keys]
3993
        elif add_prefix_to_model:
3994
3995
            expected_keys = [".".join([prefix, s]) for s in expected_keys]

3996
        missing_keys = sorted(set(expected_keys) - set(loaded_keys))
Sylvain Gugger's avatar
Sylvain Gugger committed
3997
3998
3999
4000
4001
4002
4003
4004
        unexpected_keys = set(loaded_keys) - set(expected_keys)
        # Remove nonpersistent buffers from unexpected keys: they are not in the state dict but will be in the model
        # buffers
        model_buffers = {n for n, _ in model.named_buffers()}
        if remove_prefix_from_model:
            model_buffers = {key[len(_prefix) :] if key.startswith(_prefix) else key for key in model_buffers}
        elif add_prefix_to_model:
            model_buffers = {".".join([prefix, key]) for key in model_buffers}
4005
        unexpected_keys = sorted(unexpected_keys - model_buffers)
4006

4007
        model.tie_weights()
4008
        if device_map is None and not is_fsdp_enabled() and not is_deepspeed_zero3_enabled():
4009
4010
4011
4012
            ptrs = collections.defaultdict(list)
            for name, tensor in model.state_dict().items():
                id_tensor = id_tensor_storage(tensor)
                ptrs[id_tensor].append(name)
Sylvain Gugger's avatar
Sylvain Gugger committed
4013

4014
4015
4016
4017
4018
            # These are all the pointers of shared tensors.
            tied_params = [names for _, names in ptrs.items() if len(names) > 1]
        else:
            # id function doesn't work for meta tensor so we need this function
            tied_params = find_tied_parameters(model)
Sylvain Gugger's avatar
Sylvain Gugger committed
4019
4020

        for group in tied_params:
Sylvain Gugger's avatar
Sylvain Gugger committed
4021
4022
4023
4024
            if remove_prefix_from_model:
                group = [key[len(_prefix) :] if key.startswith(_prefix) else key for key in group]
            elif add_prefix_to_model:
                group = [".".join([prefix, key]) for key in group]
Sylvain Gugger's avatar
Sylvain Gugger committed
4025
4026
4027
            missing_in_group = [k for k in missing_keys if k in group]
            if len(missing_in_group) > 0 and len(missing_in_group) < len(group):
                missing_keys = [k for k in missing_keys if k not in missing_in_group]
4028

4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
        # Some models may have keys that are not in the state by design, removing them before needlessly warning
        # the user.
        if cls._keys_to_ignore_on_load_missing is not None:
            for pat in cls._keys_to_ignore_on_load_missing:
                missing_keys = [k for k in missing_keys if re.search(pat, k) is None]

        if cls._keys_to_ignore_on_load_unexpected is not None:
            for pat in cls._keys_to_ignore_on_load_unexpected:
                unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]

4039
4040
4041
4042
        # retrieve weights on meta device and put them back on CPU.
        # This is not ideal in terms of memory, but if we don't do that not, we can't initialize them in the next step
        if low_cpu_mem_usage:
            for key in missing_keys:
Susnato Dhar's avatar
Susnato Dhar committed
4043
4044
                if key in list(model_state_dict.keys()):
                    key = key
4045
4046
                elif f"{prefix}.{key}" in list(model_state_dict.keys()):
                    key = f"{prefix}.{key}"
Susnato Dhar's avatar
Susnato Dhar committed
4047
                elif key.startswith(prefix) and ".".join(key.split(".")[1:]) in list(model_state_dict.keys()):
4048
4049
                    key = ".".join(key.split(".")[1:])
                param = model_state_dict[key]
4050
4051
4052
4053
4054
4055

                # upcast in fp32 if any
                target_dtype = dtype
                if (
                    keep_in_fp32_modules is not None
                    and dtype == torch.float16
4056
4057
4058
                    and any(
                        module_to_keep_in_fp32 in key.split(".") for module_to_keep_in_fp32 in keep_in_fp32_modules
                    )
4059
4060
4061
                ):
                    target_dtype = torch.float32

4062
                if param.device == torch.device("meta"):
4063
                    if not (is_quantized):
4064
                        set_module_tensor_to_device(model, key, "cpu", torch.empty(*param.size(), dtype=target_dtype))
4065
                    else:
4066
                        set_module_quantized_tensor_to_device(
4067
4068
                            model, key, "cpu", torch.empty(*param.size(), dtype=target_dtype)
                        )
4069
4070

        # retrieve unintialized modules and initialize before maybe overriding that with the pretrained weights.
4071
        if _fast_init:
4072
4073
4074
4075
4076
4077
4078
            if not ignore_mismatched_sizes:
                if remove_prefix_from_model:
                    _loaded_keys = [f"{prefix}.{k}" for k in loaded_keys]
                elif add_prefix_to_model:
                    _loaded_keys = [k[len(prefix) + 1 :] for k in loaded_keys]
                else:
                    _loaded_keys = loaded_keys
4079
4080
4081
                not_initialized_submodules = set_initialized_submodules(model, _loaded_keys)
            else:
                not_initialized_submodules = dict(model.named_modules())
4082
            # This will only initialize submodules that are not marked as initialized by the line above.
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
            if is_deepspeed_zero3_enabled():
                import deepspeed

                not_initialized_parameters = list(
                    set(
                        itertools.chain.from_iterable(
                            submodule.parameters(recurse=False) for submodule in not_initialized_submodules.values()
                        )
                    )
                )
                with deepspeed.zero.GatheredParameters(not_initialized_parameters, modifier_rank=0):
                    model.apply(model._initialize_weights)
            else:
                model.apply(model._initialize_weights)
4097

4098
4099
4100
        # Set some modules to fp32 if any
        if keep_in_fp32_modules is not None:
            for name, param in model.named_parameters():
4101
                if any(module_to_keep_in_fp32 in name.split(".") for module_to_keep_in_fp32 in keep_in_fp32_modules):
4102
4103
                    # param = param.to(torch.float32) does not work here as only in the local scope.
                    param.data = param.data.to(torch.float32)
4104

4105
4106
4107
        # Make sure we are able to load base models as well as derived models (with heads)
        start_prefix = ""
        model_to_load = model
4108
        if len(cls.base_model_prefix) > 0 and not hasattr(model, cls.base_model_prefix) and has_prefix_module:
4109
            start_prefix = cls.base_model_prefix + "."
4110
        if len(cls.base_model_prefix) > 0 and hasattr(model, cls.base_model_prefix) and not has_prefix_module:
4111
            model_to_load = getattr(model, cls.base_model_prefix)
Sylvain Gugger's avatar
Sylvain Gugger committed
4112
4113
            base_model_expected_keys = list(model_to_load.state_dict().keys())
            if any(key in expected_keys_not_prefixed and key not in base_model_expected_keys for key in loaded_keys):
4114
                raise ValueError(
4115
                    "The state dictionary of the model you are trying to load is corrupted. Are you sure it was "
4116
4117
                    "properly saved?"
                )
4118
4119
            if device_map is not None:
                device_map = {k.replace(f"{cls.base_model_prefix}.", ""): v for k, v in device_map.items()}
4120

4121
4122
4123
4124
4125
4126
4127
4128
        def _find_mismatched_keys(
            state_dict,
            model_state_dict,
            loaded_keys,
            add_prefix_to_model,
            remove_prefix_from_model,
            ignore_mismatched_sizes,
        ):
Sylvain Gugger's avatar
Sylvain Gugger committed
4129
4130
4131
            mismatched_keys = []
            if ignore_mismatched_sizes:
                for checkpoint_key in loaded_keys:
4132
4133
4134
                    # If the checkpoint is sharded, we may not have the key here.
                    if checkpoint_key not in state_dict:
                        continue
Sylvain Gugger's avatar
Sylvain Gugger committed
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
                    model_key = checkpoint_key
                    if remove_prefix_from_model:
                        # The model key starts with `prefix` but `checkpoint_key` doesn't so we add it.
                        model_key = f"{prefix}.{checkpoint_key}"
                    elif add_prefix_to_model:
                        # The model key doesn't start with `prefix` but `checkpoint_key` does so we remove it.
                        model_key = ".".join(checkpoint_key.split(".")[1:])

                    if (
                        model_key in model_state_dict
                        and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape
                    ):
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
                        if (
                            state_dict[checkpoint_key].shape[-1] == 1
                            and state_dict[checkpoint_key].numel() * 2 == model_state_dict[model_key].numel()
                        ):
                            # This skips size mismatches for 4-bit weights. Two 4-bit values share an 8-bit container, causing size differences.
                            # Without matching with module type or paramter type it seems like a practical way to detect valid 4bit weights.
                            pass
                        else:
                            mismatched_keys.append(
                                (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape)
                            )
                            del state_dict[checkpoint_key]
4159
4160
            return mismatched_keys

4161
4162
4163
4164
        if resolved_archive_file is not None:
            folder = os.path.sep.join(resolved_archive_file[0].split(os.path.sep)[:-1])
        else:
            folder = None
Sylvain Gugger's avatar
Sylvain Gugger committed
4165
        if device_map is not None and is_safetensors:
4166
            param_device_map = expand_device_map(device_map, original_loaded_keys, start_prefix)
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
            str_dtype = str(dtype).replace("torch.", "") if dtype is not None else "float32"
            if sharded_metadata is None:
                archive_file = (
                    resolved_archive_file[0]
                    if isinstance(resolved_archive_file, (list, tuple))
                    else resolved_archive_file
                )
                weight_map = {p: archive_file for p in original_loaded_keys}
            else:
                weight_map = {p: os.path.join(folder, f) for p, f in sharded_metadata["weight_map"].items()}
Sylvain Gugger's avatar
Sylvain Gugger committed
4177
            offload_index = {
4178
                p[len(start_prefix) :]: {"safetensors_file": f, "weight_name": p, "dtype": str_dtype}
4179
                for p, f in weight_map.items()
4180
                if p.startswith(start_prefix) and param_device_map[p[len(start_prefix) :]] == "disk"
Sylvain Gugger's avatar
Sylvain Gugger committed
4181
4182
            }

4183
4184
4185
4186
4187
        if state_dict is not None:
            # Whole checkpoint
            mismatched_keys = _find_mismatched_keys(
                state_dict,
                model_state_dict,
4188
                original_loaded_keys,
4189
4190
4191
4192
                add_prefix_to_model,
                remove_prefix_from_model,
                ignore_mismatched_sizes,
            )
Sylvain Gugger's avatar
Sylvain Gugger committed
4193
            error_msgs = _load_state_dict_into_model(model_to_load, state_dict, start_prefix)
Sylvain Gugger's avatar
Sylvain Gugger committed
4194
            offload_index = None
Sylvain Gugger's avatar
Sylvain Gugger committed
4195
        else:
4196
4197
            # Sharded checkpoint or whole but low_cpu_mem_usage==True

Sylvain Gugger's avatar
Sylvain Gugger committed
4198
4199
4200
4201
4202
            # This should always be a list but, just to be sure.
            if not isinstance(resolved_archive_file, list):
                resolved_archive_file = [resolved_archive_file]

            error_msgs = []
4203
            mismatched_keys = []
Sylvain Gugger's avatar
Sylvain Gugger committed
4204
4205
            if not is_safetensors:
                offload_index = {} if device_map is not None and "disk" in device_map.values() else None
4206
4207
4208
4209
4210
4211
4212
            if offload_state_dict:
                state_dict_folder = tempfile.mkdtemp()
                state_dict_index = {}
            else:
                state_dict_folder = None
                state_dict_index = None

4213
            if is_sharded_safetensors:
4214
4215
4216
                disk_only_shard_files = get_disk_only_shard_files(
                    device_map, sharded_metadata=sharded_metadata, start_prefix=start_prefix
                )
Sylvain Gugger's avatar
Sylvain Gugger committed
4217
4218
4219
4220
                disk_only_shard_files = [os.path.join(folder, f) for f in disk_only_shard_files]
            else:
                disk_only_shard_files = []

4221
4222
            if len(resolved_archive_file) > 1:
                resolved_archive_file = logging.tqdm(resolved_archive_file, desc="Loading checkpoint shards")
Sylvain Gugger's avatar
Sylvain Gugger committed
4223
            for shard_file in resolved_archive_file:
Sylvain Gugger's avatar
Sylvain Gugger committed
4224
4225
4226
                # Skip the load for shards that only contain disk-offloaded weights when using safetensors for the offload.
                if shard_file in disk_only_shard_files:
                    continue
Sylvain Gugger's avatar
Sylvain Gugger committed
4227
                state_dict = load_state_dict(shard_file)
4228

Sylvain Gugger's avatar
Sylvain Gugger committed
4229
4230
                # Mistmatched keys contains tuples key/shape1/shape2 of weights in the checkpoint that have a shape not
                # matching the weights in the model.
4231
4232
4233
                mismatched_keys += _find_mismatched_keys(
                    state_dict,
                    model_state_dict,
4234
                    original_loaded_keys,
4235
4236
4237
4238
                    add_prefix_to_model,
                    remove_prefix_from_model,
                    ignore_mismatched_sizes,
                )
4239
                if low_cpu_mem_usage:
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
                    if is_fsdp_enabled() and not is_local_dist_rank_0():
                        for key, param in model_to_load.state_dict().items():
                            if param.device == torch.device("meta"):
                                if not (is_quantized):
                                    set_module_tensor_to_device(
                                        model_to_load, key, "cpu", torch.empty(*param.size(), dtype=dtype)
                                    )
                                else:
                                    set_module_quantized_tensor_to_device(
                                        model_to_load, key, "cpu", torch.empty(*param.size(), dtype=dtype)
                                    )
                    else:
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
                        new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(
                            model_to_load,
                            state_dict,
                            loaded_keys,
                            start_prefix,
                            expected_keys,
                            device_map=device_map,
                            offload_folder=offload_folder,
                            offload_index=offload_index,
                            state_dict_folder=state_dict_folder,
                            state_dict_index=state_dict_index,
                            dtype=dtype,
                            is_quantized=is_quantized,
                            is_safetensors=is_safetensors,
                            keep_in_fp32_modules=keep_in_fp32_modules,
4267
                            unexpected_keys=unexpected_keys,
4268
4269
                        )
                        error_msgs += new_error_msgs
4270
4271
                else:
                    error_msgs += _load_state_dict_into_model(model_to_load, state_dict, start_prefix)
4272

4273
4274
4275
4276
                # force memory release
                del state_dict
                gc.collect()

4277
            if offload_index is not None and len(offload_index) > 0:
Sylvain Gugger's avatar
Sylvain Gugger committed
4278
4279
4280
                if model != model_to_load:
                    # We need to add the prefix of the base model
                    prefix = cls.base_model_prefix
Sylvain Gugger's avatar
Sylvain Gugger committed
4281
4282
4283
4284
4285
4286
                    if not is_safetensors:
                        for weight_name in offload_index:
                            shutil.move(
                                os.path.join(offload_folder, f"{weight_name}.dat"),
                                os.path.join(offload_folder, f"{prefix}.{weight_name}.dat"),
                            )
Sylvain Gugger's avatar
Sylvain Gugger committed
4287
                    offload_index = {f"{prefix}.{key}": value for key, value in offload_index.items()}
Sylvain Gugger's avatar
Sylvain Gugger committed
4288
4289
4290
                if not is_safetensors:
                    save_offload_index(offload_index, offload_folder)
                    offload_index = None
4291
4292
4293

            if offload_state_dict:
                # Load back temporarily offloaded state dict
4294
                load_offloaded_weights(model_to_load, state_dict_index, state_dict_folder)
4295
4296
                shutil.rmtree(state_dict_folder)

4297
4298
        if len(error_msgs) > 0:
            error_msg = "\n\t".join(error_msgs)
4299
4300
4301
4302
            if "size mismatch" in error_msg:
                error_msg += (
                    "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method."
                )
4303
4304
            raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}")

4305
        if len(unexpected_keys) > 0:
Sylvain Gugger's avatar
Sylvain Gugger committed
4306
            archs = [] if model.config.architectures is None else model.config.architectures
4307
            warner = logger.warning if model.__class__.__name__ in archs else logger.info
Sylvain Gugger's avatar
Sylvain Gugger committed
4308
            warner(
Sylvain Gugger's avatar
Sylvain Gugger committed
4309
4310
4311
4312
4313
4314
4315
                f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"
                f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"
                f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or"
                " with another architecture (e.g. initializing a BertForSequenceClassification model from a"
                " BertForPreTraining model).\n- This IS NOT expected if you are initializing"
                f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly identical"
                " (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)."
4316
4317
4318
4319
4320
            )
        else:
            logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n")
        if len(missing_keys) > 0:
            logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
4321
4322
4323
                f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"
                " TRAIN this model on a down-stream task to be able to use it for predictions and inference."
4324
            )
4325
        elif len(mismatched_keys) == 0:
4326
            logger.info(
Sylvain Gugger's avatar
Sylvain Gugger committed
4327
4328
4329
4330
                f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the checkpoint"
                f" was trained on, you can already use {model.__class__.__name__} for predictions without further"
                " training."
4331
            )
4332
4333
4334
4335
4336
4337
4338
4339
        if len(mismatched_keys) > 0:
            mismatched_warning = "\n".join(
                [
                    f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated"
                    for key, shape1, shape2 in mismatched_keys
                ]
            )
            logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
4340
4341
4342
4343
                f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not"
                f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be able"
                " to use it for predictions and inference."
4344
            )
4345

Sylvain Gugger's avatar
Sylvain Gugger committed
4346
        return model, missing_keys, unexpected_keys, mismatched_keys, offload_index, error_msgs
4347
4348

    def retrieve_modules_from_names(self, names, add_prefix=False, remove_prefix=False):
4349
        module_keys = {".".join(key.split(".")[:-1]) for key in names}
4350

Patrick von Platen's avatar
Patrick von Platen committed
4351
4352
        # torch.nn.ParameterList is a special case where two parameter keywords
        # are appended to the module name, *e.g.* bert.special_embeddings.0
4353
        module_keys = module_keys.union(
4354
            {".".join(key.split(".")[:-2]) for key in names if len(key) > 0 and key[-1].isdigit()}
4355
        )
Patrick von Platen's avatar
Patrick von Platen committed
4356

4357
4358
4359
4360
        retrieved_modules = []
        # retrieve all modules that has at least one missing weight name
        for name, module in self.named_modules():
            if remove_prefix:
4361
4362
                _prefix = f"{self.base_model_prefix}."
                name = name[len(_prefix) :] if name.startswith(_prefix) else name
4363
            elif add_prefix:
Patrick von Platen's avatar
Patrick von Platen committed
4364
                name = ".".join([self.base_model_prefix, name]) if len(name) > 0 else self.base_model_prefix
4365
4366
4367
4368
4369
4370

            if name in module_keys:
                retrieved_modules.append(module)

        return retrieved_modules

4371
    @staticmethod
4372
    def _load_pretrained_model_low_mem(model, loaded_state_dict_keys, resolved_archive_file, start_prefix=""):
4373
4374
4375
        """
        This is an experimental function that loads the model using ~1.x model size CPU memory

4376
        Before you call it do:
4377

4378
        1. save which state_dict keys are available
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
        2. drop state_dict before model is created, since the latter takes 1x model size memory

        Here then we continue:

        3. switch to the meta device all params/buffers that are going to be replaced from the loaded state_dict
        4. load state_dict 2nd time
        5. replace the params/buffers from the state_dict

        Currently, it doesn't handle missing_keys, unexpected_keys, mismatched_keys. It can't handle deepspeed.
        """

4390
4391
4392
4393
        _move_model_to_meta(model, loaded_state_dict_keys, start_prefix)
        state_dict = load_state_dict(resolved_archive_file)
        error_msgs = _load_state_dict_into_meta_model(model, state_dict, loaded_state_dict_keys, start_prefix)
        return error_msgs
4394

4395
4396
4397
4398
4399
4400
    @classmethod
    def register_for_auto_class(cls, auto_class="AutoModel"):
        """
        Register this class with a given auto class. This should only be used for custom models as the ones in the
        library are already mapped with an auto class.

4401
4402
4403
4404
4405
4406
        <Tip warning={true}>

        This API is experimental and may have some slight breaking changes in the next releases.

        </Tip>

4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
        Args:
            auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`):
                The auto class to register this new model with.
        """
        if not isinstance(auto_class, str):
            auto_class = auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(auto_module, auto_class):
            raise ValueError(f"{auto_class} is not a valid auto class.")

        cls._auto_class = auto_class

4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
    def to_bettertransformer(self) -> "PreTrainedModel":
        """
        Converts the model to use [PyTorch's native attention
        implementation](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html), integrated to
        Transformers through [Optimum library](https://huggingface.co/docs/optimum/bettertransformer/overview). Only a
        subset of all Transformers models are supported.

        PyTorch's attention fastpath allows to speed up inference through kernel fusions and the use of [nested
        tensors](https://pytorch.org/docs/stable/nested.html). Detailed benchmarks can be found in [this blog
        post](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2).

        Returns:
            [`PreTrainedModel`]: The model converted to BetterTransformer.
        """
        if not is_optimum_available():
            raise ImportError("The package `optimum` is required to use Better Transformer.")

        from optimum.version import __version__ as optimum_version

        if version.parse(optimum_version) < version.parse("1.7.0"):
            raise ImportError(
                f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
            )

        from optimum.bettertransformer import BetterTransformer

        return BetterTransformer.transform(self)

    def reverse_bettertransformer(self):
        """
        Reverts the transformation from [`~PreTrainedModel.to_bettertransformer`] so that the original modeling is
        used, for example in order to save the model.

        Returns:
            [`PreTrainedModel`]: The model converted back to the original modeling.
        """
        if not is_optimum_available():
            raise ImportError("The package `optimum` is required to use Better Transformer.")

        from optimum.version import __version__ as optimum_version

        if version.parse(optimum_version) < version.parse("1.7.0"):
            raise ImportError(
                f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
            )

        from optimum.bettertransformer import BetterTransformer

        return BetterTransformer.reverse(self)

4471
4472
4473
4474
    def warn_if_padding_and_no_attention_mask(self, input_ids, attention_mask):
        """
        Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given.
        """
4475
4476

        # Skip the check during tracing.
4477
        if is_torch_fx_proxy(input_ids) or torch.jit.is_tracing() or is_torchdynamo_compiling():
4478
4479
            return

4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
        if (attention_mask is not None) or (self.config.pad_token_id is None):
            return

        # Check only the first and last input IDs to reduce overhead.
        if self.config.pad_token_id in input_ids[:, [-1, 0]]:
            warn_string = (
                "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See "
                "https://huggingface.co/docs/transformers/troubleshooting"
                "#incorrect-output-when-padding-tokens-arent-masked."
            )

            # If the pad token is equal to either BOS, EOS, or SEP, we do not know whether the user should use an
            # attention_mask or not. In this case, we should still show a warning because this is a rare case.
            if (
                (self.config.bos_token_id is not None and self.config.bos_token_id == self.config.pad_token_id)
                or (self.config.eos_token_id is not None and self.config.eos_token_id == self.config.pad_token_id)
                or (self.config.sep_token_id is not None and self.config.sep_token_id == self.config.pad_token_id)
            ):
                warn_string += (
                    f"\nYou may ignore this warning if your `pad_token_id` ({self.config.pad_token_id}) is identical "
                    f"to the `bos_token_id` ({self.config.bos_token_id}), `eos_token_id` ({self.config.eos_token_id}), "
                    f"or the `sep_token_id` ({self.config.sep_token_id}), and your input is not padded."
                )

            logger.warning_once(warn_string)

thomwolf's avatar
thomwolf committed
4506

4507
PreTrainedModel.push_to_hub = copy_func(PreTrainedModel.push_to_hub)
4508
4509
4510
4511
if PreTrainedModel.push_to_hub.__doc__ is not None:
    PreTrainedModel.push_to_hub.__doc__ = PreTrainedModel.push_to_hub.__doc__.format(
        object="model", object_class="AutoModel", object_files="model file"
    )
4512
4513


thomwolf's avatar
thomwolf committed
4514
class PoolerStartLogits(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4515
4516
    """
    Compute SQuAD start logits from sequence hidden states.
4517

Sylvain Gugger's avatar
Sylvain Gugger committed
4518
    Args:
4519
4520
        config ([`PretrainedConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
Sylvain Gugger's avatar
Sylvain Gugger committed
4521
4522
4523
    """

    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
4524
        super().__init__()
thomwolf's avatar
thomwolf committed
4525
4526
        self.dense = nn.Linear(config.hidden_size, 1)

Sylvain Gugger's avatar
Sylvain Gugger committed
4527
4528
4529
4530
4531
    def forward(
        self, hidden_states: torch.FloatTensor, p_mask: Optional[torch.FloatTensor] = None
    ) -> torch.FloatTensor:
        """
        Args:
4532
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4533
                The final hidden states of the model.
4534
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4535
4536
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
Sylvain Gugger's avatar
Sylvain Gugger committed
4537
4538

        Returns:
4539
            `torch.FloatTensor`: The start logits for SQuAD.
thomwolf's avatar
thomwolf committed
4540
        """
thomwolf's avatar
thomwolf committed
4541
4542
4543
        x = self.dense(hidden_states).squeeze(-1)

        if p_mask is not None:
Lysandre Debut's avatar
Lysandre Debut committed
4544
            if get_parameter_dtype(self) == torch.float16:
4545
4546
4547
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
4548
4549
4550
4551
4552
4553

        return x


class PoolerEndLogits(nn.Module):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
4554
    Compute SQuAD end logits from sequence hidden states.
4555

Sylvain Gugger's avatar
Sylvain Gugger committed
4556
    Args:
4557
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4558
4559
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
Sylvain Gugger's avatar
Sylvain Gugger committed
4560
4561
4562
    """

    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
4563
        super().__init__()
thomwolf's avatar
thomwolf committed
4564
4565
4566
4567
4568
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dense_1 = nn.Linear(config.hidden_size, 1)

Sylvain Gugger's avatar
Sylvain Gugger committed
4569
4570
4571
4572
4573
4574
4575
4576
4577
    def forward(
        self,
        hidden_states: torch.FloatTensor,
        start_states: Optional[torch.FloatTensor] = None,
        start_positions: Optional[torch.LongTensor] = None,
        p_mask: Optional[torch.FloatTensor] = None,
    ) -> torch.FloatTensor:
        """
        Args:
4578
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4579
                The final hidden states of the model.
4580
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4581
                The hidden states of the first tokens for the labeled span.
4582
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4583
                The position of the first token for the labeled span.
4584
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4585
4586
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
Sylvain Gugger's avatar
Sylvain Gugger committed
4587

4588
        <Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4589

Stas Bekman's avatar
Stas Bekman committed
4590
4591
        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.
4592
4593

        </Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4594
4595

        Returns:
4596
            `torch.FloatTensor`: The end logits for SQuAD.
thomwolf's avatar
thomwolf committed
4597
        """
4598
4599
4600
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
4601
        if start_positions is not None:
4602
            slen, hsz = hidden_states.shape[-2:]
4603
4604
4605
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions)  # shape (bsz, 1, hsz)
            start_states = start_states.expand(-1, slen, -1)  # shape (bsz, slen, hsz)
thomwolf's avatar
thomwolf committed
4606
4607
4608
4609
4610
4611
4612

        x = self.dense_0(torch.cat([hidden_states, start_states], dim=-1))
        x = self.activation(x)
        x = self.LayerNorm(x)
        x = self.dense_1(x).squeeze(-1)

        if p_mask is not None:
Lysandre Debut's avatar
Lysandre Debut committed
4613
            if get_parameter_dtype(self) == torch.float16:
4614
4615
4616
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
4617
4618
4619
4620
4621

        return x


class PoolerAnswerClass(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4622
4623
4624
4625
    """
    Compute SQuAD 2.0 answer class from classification and start tokens hidden states.

    Args:
4626
4627
        config ([`PretrainedConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
Sylvain Gugger's avatar
Sylvain Gugger committed
4628
    """
4629

thomwolf's avatar
thomwolf committed
4630
    def __init__(self, config):
Julien Chaumond's avatar
Julien Chaumond committed
4631
        super().__init__()
thomwolf's avatar
thomwolf committed
4632
4633
4634
4635
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.dense_1 = nn.Linear(config.hidden_size, 1, bias=False)

Sylvain Gugger's avatar
Sylvain Gugger committed
4636
4637
4638
4639
4640
4641
4642
    def forward(
        self,
        hidden_states: torch.FloatTensor,
        start_states: Optional[torch.FloatTensor] = None,
        start_positions: Optional[torch.LongTensor] = None,
        cls_index: Optional[torch.LongTensor] = None,
    ) -> torch.FloatTensor:
4643
4644
        """
        Args:
4645
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4646
                The final hidden states of the model.
4647
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4648
                The hidden states of the first tokens for the labeled span.
4649
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4650
                The position of the first token for the labeled span.
4651
4652
4653
4654
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.

        <Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4655

Stas Bekman's avatar
Stas Bekman committed
4656
4657
        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.
Sylvain Gugger's avatar
Sylvain Gugger committed
4658

4659
        </Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4660
4661

        Returns:
4662
            `torch.FloatTensor`: The SQuAD 2.0 answer class.
thomwolf's avatar
thomwolf committed
4663
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
4664
        # No dependency on end_feature so that we can obtain one single `cls_logits` for each sample.
4665
        hsz = hidden_states.shape[-1]
4666
4667
4668
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
4669
        if start_positions is not None:
4670
4671
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
4672
4673

        if cls_index is not None:
4674
4675
            cls_index = cls_index[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            cls_token_state = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
4676
        else:
4677
            cls_token_state = hidden_states[:, -1, :]  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
4678
4679
4680
4681
4682
4683
4684
4685

        x = self.dense_0(torch.cat([start_states, cls_token_state], dim=-1))
        x = self.activation(x)
        x = self.dense_1(x).squeeze(-1)

        return x


4686
4687
4688
@dataclass
class SquadHeadOutput(ModelOutput):
    """
4689
    Base class for outputs of question answering models using a [`~modeling_utils.SQuADHead`].
4690
4691

    Args:
4692
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
Sylvain Gugger's avatar
Sylvain Gugger committed
4693
4694
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification
            losses.
4695
        start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
4696
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
4697
        start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
4698
            Indices for the top config.start_n_top start token possibilities (beam-search).
4699
4700
        end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
Sylvain Gugger's avatar
Sylvain Gugger committed
4701
            (beam-search).
4702
4703
4704
4705
        end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
        cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the `is_impossible` label of the answers.
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716

    """

    loss: Optional[torch.FloatTensor] = None
    start_top_log_probs: Optional[torch.FloatTensor] = None
    start_top_index: Optional[torch.LongTensor] = None
    end_top_log_probs: Optional[torch.FloatTensor] = None
    end_top_index: Optional[torch.LongTensor] = None
    cls_logits: Optional[torch.FloatTensor] = None


thomwolf's avatar
thomwolf committed
4717
class SQuADHead(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4718
4719
    r"""
    A SQuAD head inspired by XLNet.
4720

Sylvain Gugger's avatar
Sylvain Gugger committed
4721
    Args:
4722
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4723
4724
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
thomwolf's avatar
thomwolf committed
4725
    """
4726

thomwolf's avatar
thomwolf committed
4727
    def __init__(self, config):
Julien Chaumond's avatar
Julien Chaumond committed
4728
        super().__init__()
thomwolf's avatar
thomwolf committed
4729
4730
4731
4732
4733
4734
4735
        self.start_n_top = config.start_n_top
        self.end_n_top = config.end_n_top

        self.start_logits = PoolerStartLogits(config)
        self.end_logits = PoolerEndLogits(config)
        self.answer_class = PoolerAnswerClass(config)

Sylvain Gugger's avatar
Sylvain Gugger committed
4736
    @replace_return_docstrings(output_type=SquadHeadOutput, config_class=PretrainedConfig)
4737
    def forward(
4738
        self,
Sylvain Gugger's avatar
Sylvain Gugger committed
4739
4740
4741
4742
4743
4744
        hidden_states: torch.FloatTensor,
        start_positions: Optional[torch.LongTensor] = None,
        end_positions: Optional[torch.LongTensor] = None,
        cls_index: Optional[torch.LongTensor] = None,
        is_impossible: Optional[torch.LongTensor] = None,
        p_mask: Optional[torch.FloatTensor] = None,
4745
        return_dict: bool = False,
Sylvain Gugger's avatar
Sylvain Gugger committed
4746
4747
    ) -> Union[SquadHeadOutput, Tuple[torch.FloatTensor]]:
        """
Lysandre's avatar
Lysandre committed
4748
        Args:
4749
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Lysandre's avatar
Lysandre committed
4750
                Final hidden states of the model on the sequence tokens.
4751
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4752
                Positions of the first token for the labeled span.
4753
            end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4754
                Positions of the last token for the labeled span.
4755
4756
4757
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.
            is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4758
                Whether the question has a possible answer in the paragraph or not.
4759
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4760
4761
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
4762
            return_dict (`bool`, *optional*, defaults to `False`):
4763
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
Sylvain Gugger's avatar
Sylvain Gugger committed
4764

Lysandre's avatar
Lysandre committed
4765
        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
4766
        """
thomwolf's avatar
thomwolf committed
4767
        start_logits = self.start_logits(hidden_states, p_mask=p_mask)
thomwolf's avatar
thomwolf committed
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790

        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, let's remove the dimension added by batch splitting
            for x in (start_positions, end_positions, cls_index, is_impossible):
                if x is not None and x.dim() > 1:
                    x.squeeze_(-1)

            # during training, compute the end logits based on the ground truth of the start position
            end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask)

            loss_fct = CrossEntropyLoss()
            start_loss = loss_fct(start_logits, start_positions)
            end_loss = loss_fct(end_logits, end_positions)
            total_loss = (start_loss + end_loss) / 2

            if cls_index is not None and is_impossible is not None:
                # Predict answerability from the representation of CLS and START
                cls_logits = self.answer_class(hidden_states, start_positions=start_positions, cls_index=cls_index)
                loss_fct_cls = nn.BCEWithLogitsLoss()
                cls_loss = loss_fct_cls(cls_logits, is_impossible)

                # note(zhiliny): by default multiply the loss by 0.5 so that the scale is comparable to start_loss and end_loss
                total_loss += cls_loss * 0.5
4791

4792
            return SquadHeadOutput(loss=total_loss) if return_dict else (total_loss,)
thomwolf's avatar
thomwolf committed
4793
4794
4795
4796

        else:
            # during inference, compute the end logits based on beam search
            bsz, slen, hsz = hidden_states.size()
4797
            start_log_probs = nn.functional.softmax(start_logits, dim=-1)  # shape (bsz, slen)
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808

            start_top_log_probs, start_top_index = torch.topk(
                start_log_probs, self.start_n_top, dim=-1
            )  # shape (bsz, start_n_top)
            start_top_index_exp = start_top_index.unsqueeze(-1).expand(-1, -1, hsz)  # shape (bsz, start_n_top, hsz)
            start_states = torch.gather(hidden_states, -2, start_top_index_exp)  # shape (bsz, start_n_top, hsz)
            start_states = start_states.unsqueeze(1).expand(-1, slen, -1, -1)  # shape (bsz, slen, start_n_top, hsz)

            hidden_states_expanded = hidden_states.unsqueeze(2).expand_as(
                start_states
            )  # shape (bsz, slen, start_n_top, hsz)
thomwolf's avatar
thomwolf committed
4809
4810
            p_mask = p_mask.unsqueeze(-1) if p_mask is not None else None
            end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask)
4811
            end_log_probs = nn.functional.softmax(end_logits, dim=1)  # shape (bsz, slen, start_n_top)
thomwolf's avatar
thomwolf committed
4812

4813
4814
4815
            end_top_log_probs, end_top_index = torch.topk(
                end_log_probs, self.end_n_top, dim=1
            )  # shape (bsz, end_n_top, start_n_top)
thomwolf's avatar
thomwolf committed
4816
4817
4818
4819
4820
4821
            end_top_log_probs = end_top_log_probs.view(-1, self.start_n_top * self.end_n_top)
            end_top_index = end_top_index.view(-1, self.start_n_top * self.end_n_top)

            start_states = torch.einsum("blh,bl->bh", hidden_states, start_log_probs)
            cls_logits = self.answer_class(hidden_states, start_states=start_states, cls_index=cls_index)

4822
            if not return_dict:
4823
4824
4825
4826
4827
4828
4829
4830
4831
                return (start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits)
            else:
                return SquadHeadOutput(
                    start_top_log_probs=start_top_log_probs,
                    start_top_index=start_top_index,
                    end_top_log_probs=end_top_log_probs,
                    end_top_index=end_top_index,
                    cls_logits=cls_logits,
                )
thomwolf's avatar
thomwolf committed
4832
4833
4834


class SequenceSummary(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4835
4836
4837
4838
    r"""
    Compute a single vector summary of a sequence hidden states.

    Args:
4839
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4840
4841
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):
Sylvain Gugger's avatar
Sylvain Gugger committed
4842

4843
            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:
Sylvain Gugger's avatar
Sylvain Gugger committed
4844

4845
4846
4847
4848
4849
                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention
Sylvain Gugger's avatar
Sylvain Gugger committed
4850

4851
            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
Sylvain Gugger's avatar
Sylvain Gugger committed
4852
4853
4854
4855
4856
4857
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
thomwolf's avatar
thomwolf committed
4858
    """
4859

4860
    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
4861
        super().__init__()
thomwolf's avatar
thomwolf committed
4862

4863
        self.summary_type = getattr(config, "summary_type", "last")
4864
        if self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
4865
4866
4867
4868
4869
            # We should use a standard multi-head attention module with absolute positional embedding for that.
            # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276
            # We can probably just use the multi-head attention module of PyTorch >=1.1.0
            raise NotImplementedError

thomwolf's avatar
thomwolf committed
4870
        self.summary = Identity()
4871
4872
        if hasattr(config, "summary_use_proj") and config.summary_use_proj:
            if hasattr(config, "summary_proj_to_labels") and config.summary_proj_to_labels and config.num_labels > 0:
4873
                num_classes = config.num_labels
thomwolf's avatar
thomwolf committed
4874
4875
4876
4877
            else:
                num_classes = config.hidden_size
            self.summary = nn.Linear(config.hidden_size, num_classes)

4878
        activation_string = getattr(config, "summary_activation", None)
Lysandre's avatar
Lysandre committed
4879
        self.activation: Callable = get_activation(activation_string) if activation_string else Identity()
thomwolf's avatar
thomwolf committed
4880

thomwolf's avatar
thomwolf committed
4881
        self.first_dropout = Identity()
4882
        if hasattr(config, "summary_first_dropout") and config.summary_first_dropout > 0:
4883
4884
            self.first_dropout = nn.Dropout(config.summary_first_dropout)

thomwolf's avatar
thomwolf committed
4885
        self.last_dropout = Identity()
4886
        if hasattr(config, "summary_last_dropout") and config.summary_last_dropout > 0:
4887
            self.last_dropout = nn.Dropout(config.summary_last_dropout)
thomwolf's avatar
thomwolf committed
4888

Sylvain Gugger's avatar
Sylvain Gugger committed
4889
4890
4891
4892
4893
4894
4895
    def forward(
        self, hidden_states: torch.FloatTensor, cls_index: Optional[torch.LongTensor] = None
    ) -> torch.FloatTensor:
        """
        Compute a single vector summary of a sequence hidden states.

        Args:
4896
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4897
                The hidden states of the last layer.
4898
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4899
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.
Sylvain Gugger's avatar
Sylvain Gugger committed
4900
4901

        Returns:
4902
            `torch.FloatTensor`: The summary of the sequence hidden states.
thomwolf's avatar
thomwolf committed
4903
        """
4904
        if self.summary_type == "last":
thomwolf's avatar
thomwolf committed
4905
            output = hidden_states[:, -1]
4906
        elif self.summary_type == "first":
thomwolf's avatar
thomwolf committed
4907
            output = hidden_states[:, 0]
4908
        elif self.summary_type == "mean":
thomwolf's avatar
thomwolf committed
4909
            output = hidden_states.mean(dim=1)
4910
        elif self.summary_type == "cls_index":
thomwolf's avatar
thomwolf committed
4911
            if cls_index is None:
Lysandre's avatar
Lysandre committed
4912
4913
4914
4915
4916
                cls_index = torch.full_like(
                    hidden_states[..., :1, :],
                    hidden_states.shape[-2] - 1,
                    dtype=torch.long,
                )
thomwolf's avatar
thomwolf committed
4917
            else:
thomwolf's avatar
thomwolf committed
4918
                cls_index = cls_index.unsqueeze(-1).unsqueeze(-1)
4919
                cls_index = cls_index.expand((-1,) * (cls_index.dim() - 1) + (hidden_states.size(-1),))
thomwolf's avatar
thomwolf committed
4920
            # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states
4921
4922
            output = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, XX, hidden_size)
        elif self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
4923
4924
            raise NotImplementedError

4925
        output = self.first_dropout(output)
thomwolf's avatar
thomwolf committed
4926
4927
        output = self.summary(output)
        output = self.activation(output)
4928
        output = self.last_dropout(output)
thomwolf's avatar
thomwolf committed
4929
4930
4931
4932

        return output


4933
def unwrap_model(model: nn.Module) -> nn.Module:
4934
4935
4936
4937
    """
    Recursively unwraps a model from potential containers (as used in distributed training).

    Args:
4938
        model (`torch.nn.Module`): The model to unwrap.
4939
4940
4941
4942
4943
4944
    """
    # since there could be multiple levels of wrapping, unwrap recursively
    if hasattr(model, "module"):
        return unwrap_model(model.module)
    else:
        return model
Sylvain Gugger's avatar
Sylvain Gugger committed
4945
4946


4947
def expand_device_map(device_map, param_names, start_prefix):
Sylvain Gugger's avatar
Sylvain Gugger committed
4948
4949
4950
4951
    """
    Expand a device map to return the correspondance parameter name to device.
    """
    new_device_map = {}
4952
    param_names = [p[len(start_prefix) :] for p in param_names if p.startswith(start_prefix)]
Sylvain Gugger's avatar
Sylvain Gugger committed
4953
    for module, device in device_map.items():
4954
4955
4956
        new_device_map.update(
            {p: device for p in param_names if p == module or p.startswith(f"{module}.") or module == ""}
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
4957
4958
4959
    return new_device_map


4960
def get_disk_only_shard_files(device_map, sharded_metadata, start_prefix):
Sylvain Gugger's avatar
Sylvain Gugger committed
4961
4962
4963
    """
    Returns the list of shard files containing only weights offloaded to disk.
    """
4964
4965
4966
4967

    weight_map = {
        p[len(start_prefix) :]: v for p, v in sharded_metadata["weight_map"].items() if p.startswith(start_prefix)
    }
Sylvain Gugger's avatar
Sylvain Gugger committed
4968
    files_content = collections.defaultdict(list)
4969
    for weight_name, filename in weight_map.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
4970
4971
4972
4973
4974
        while len(weight_name) > 0 and weight_name not in device_map:
            weight_name = ".".join(weight_name.split(".")[:-1])
        files_content[filename].append(device_map[weight_name])

    return [fname for fname, devices in files_content.items() if set(devices) == {"disk"}]