modeling_utils.py 249 KB
Newer Older
1
# coding=utf-8
thomwolf's avatar
thomwolf committed
2
# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team.
3
4
5
6
7
8
9
10
11
12
13
14
15
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
16
import collections
17
import copy
18
import functools
19
import gc
20
import importlib.metadata
Yih-Dar's avatar
Yih-Dar committed
21
import inspect
22
import itertools
Sylvain Gugger's avatar
Sylvain Gugger committed
23
import json
24
import os
25
import re
Sylvain Gugger's avatar
Sylvain Gugger committed
26
27
import shutil
import tempfile
28
import warnings
29
from contextlib import contextmanager
30
from dataclasses import dataclass
31
from functools import partial, wraps
32
from threading import Thread
33
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
34
from zipfile import is_zipfile
35
36

import torch
37
from huggingface_hub import split_torch_state_dict_into_shards
38
from packaging import version
Sylvain Gugger's avatar
Sylvain Gugger committed
39
from torch import Tensor, nn
40
from torch.nn import CrossEntropyLoss, Identity
41
from torch.utils.checkpoint import checkpoint
42

43
from .activations import get_activation
44
from .configuration_utils import PretrainedConfig
45
from .dynamic_module_utils import custom_object_save
46
from .generation import GenerationConfig, GenerationMixin
47
from .integrations import PeftAdapterMixin, deepspeed_config, is_deepspeed_zero3_enabled
48
49
50
51
from .pytorch_utils import (  # noqa: F401
    Conv1D,
    apply_chunking_to_forward,
    find_pruneable_heads_and_indices,
Thomas Wang's avatar
Thomas Wang committed
52
    id_tensor_storage,
53
    is_torch_greater_or_equal_than_1_13,
54
55
56
57
    prune_conv1d_layer,
    prune_layer,
    prune_linear_layer,
)
58
from .quantizers import AutoHfQuantizer, HfQuantizer
59
from .quantizers.quantizers_utils import get_module_from_name
60
from .safetensors_conversion import auto_conversion
61
from .utils import (
62
    ACCELERATE_MIN_VERSION,
63
64
    ADAPTER_SAFE_WEIGHTS_NAME,
    ADAPTER_WEIGHTS_NAME,
65
    CONFIG_NAME,
Aymeric Augustin's avatar
Aymeric Augustin committed
66
    DUMMY_INPUTS,
67
    FLAX_WEIGHTS_NAME,
68
69
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
70
71
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
Sylvain Gugger's avatar
Sylvain Gugger committed
72
    WEIGHTS_INDEX_NAME,
73
    WEIGHTS_NAME,
74
    ContextManagers,
75
    ModelOutput,
Sylvain Gugger's avatar
Sylvain Gugger committed
76
    PushToHubMixin,
77
    cached_file,
78
    copy_func,
79
    download_url,
80
    extract_commit_hash,
81
    has_file,
82
    is_accelerate_available,
83
    is_bitsandbytes_available,
84
    is_flash_attn_2_available,
85
    is_offline_mode,
86
    is_optimum_available,
87
    is_peft_available,
88
    is_remote_url,
89
    is_safetensors_available,
90
    is_torch_sdpa_available,
91
    is_torch_xla_available,
92
    logging,
Sylvain Gugger's avatar
Sylvain Gugger committed
93
    replace_return_docstrings,
94
    strtobool,
95
)
96
from .utils.hub import convert_file_size_to_int, create_and_tag_model_card, get_checkpoint_shard_files
97
98
99
100
101
102
from .utils.import_utils import (
    ENV_VARS_TRUE_VALUES,
    is_sagemaker_mp_enabled,
    is_torch_fx_proxy,
    is_torchdynamo_compiling,
)
103
from .utils.quantization_config import BitsAndBytesConfig, QuantizationMethod
104

Aymeric Augustin's avatar
Aymeric Augustin committed
105

106
107
XLA_USE_BF16 = os.environ.get("XLA_USE_BF16", "0").upper()
XLA_DOWNCAST_BF16 = os.environ.get("XLA_DOWNCAST_BF16", "0").upper()
108
109
PARAM_RENAME_WARNING = "A parameter name that contains `{}` will be renamed internally to `{}`. Please use a different name to suppress this warning."

110

111
112
if is_accelerate_available():
    from accelerate import dispatch_model, infer_auto_device_map, init_empty_weights
113
    from accelerate.hooks import add_hook_to_module
114
    from accelerate.utils import (
115
        check_tied_parameters_on_same_device,
116
        extract_model_from_parallel,
117
        find_tied_parameters,
118
        get_balanced_memory,
Marc Sun's avatar
Marc Sun committed
119
        get_max_memory,
120
121
122
123
124
125
        load_offloaded_weights,
        offload_weight,
        save_offload_index,
        set_module_tensor_to_device,
    )

126
127
128
129
    accelerate_version = version.parse(importlib.metadata.version("accelerate"))
    if accelerate_version >= version.parse("0.31"):
        from accelerate.utils.modeling import get_state_dict_from_offload

130
131
132
133
if is_safetensors_available():
    from safetensors import safe_open
    from safetensors.torch import load_file as safe_load_file
    from safetensors.torch import save_file as safe_save_file
134

Lysandre Debut's avatar
Lysandre Debut committed
135
logger = logging.get_logger(__name__)
136

137
138
139
140

_init_weights = True


141
def is_fsdp_enabled():
142
143
144
145
    return (
        torch.distributed.is_available()
        and torch.distributed.is_initialized()
        and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
146
        and strtobool(os.environ.get("FSDP_CPU_RAM_EFFICIENT_LOADING", "False")) == 1
147
    )
148
149


150
151
152
153
154
155
def is_local_dist_rank_0():
    return (
        torch.distributed.is_available()
        and torch.distributed.is_initialized()
        and int(os.environ.get("LOCAL_RANK", -1)) == 0
    )
156
157


158
159
160
161
162
163
164
165
if is_sagemaker_mp_enabled():
    import smdistributed.modelparallel.torch as smp
    from smdistributed.modelparallel import __version__ as SMP_VERSION

    IS_SAGEMAKER_MP_POST_1_10 = version.parse(SMP_VERSION) >= version.parse("1.10")
else:
    IS_SAGEMAKER_MP_POST_1_10 = False

166
167
168
if is_peft_available():
    from .utils import find_adapter_config_file

169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
TORCH_INIT_FUNCTIONS = {
    "uniform_": nn.init.uniform_,
    "normal_": nn.init.normal_,
    "trunc_normal_": nn.init.trunc_normal_,
    "constant_": nn.init.constant_,
    "xavier_uniform_": nn.init.xavier_uniform_,
    "xavier_normal_": nn.init.xavier_normal_,
    "kaiming_uniform_": nn.init.kaiming_uniform_,
    "kaiming_normal_": nn.init.kaiming_normal_,
    "uniform": nn.init.uniform,
    "normal": nn.init.normal,
    "xavier_uniform": nn.init.xavier_uniform,
    "xavier_normal": nn.init.xavier_normal,
    "kaiming_uniform": nn.init.kaiming_uniform,
    "kaiming_normal": nn.init.kaiming_normal,
}

186

187
188
189
190
191
192
193
194
@contextmanager
def no_init_weights(_enable=True):
    """
    Context manager to globally disable weight initialization to speed up loading large models.

    TODO(Patrick): Delete safety argument `_enable=True` at next major version. .
    """
    global _init_weights
195
    old_init_weights = _init_weights
196

197
198
    if _enable:
        _init_weights = False
199
200
201
202
203
204
205

        def _skip_init(*args, **kwargs):
            pass

        # # Save the original initialization functions
        for name, init_func in TORCH_INIT_FUNCTIONS.items():
            setattr(torch.nn.init, name, _skip_init)
206
207
208
    try:
        yield
    finally:
209
        _init_weights = old_init_weights
210
211
212
213
        if _enable:
            # # Restore the original initialization functions
            for name, init_func in TORCH_INIT_FUNCTIONS.items():
                setattr(torch.nn.init, name, init_func)
214
215


Lysandre Debut's avatar
Lysandre Debut committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
def get_parameter_device(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    try:
        return next(parameter.parameters()).device
    except StopIteration:
        # For nn.DataParallel compatibility in PyTorch 1.5

        def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
            tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
            return tuples

        gen = parameter._named_members(get_members_fn=find_tensor_attributes)
        first_tuple = next(gen)
        return first_tuple[1].device


231
232
233
234
def get_first_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    """
    Returns the first parameter dtype (can be non-floating) or asserts if none were found.
    """
Lysandre Debut's avatar
Lysandre Debut committed
235
236
237
    try:
        return next(parameter.parameters()).dtype
    except StopIteration:
Sylvain Gugger's avatar
Sylvain Gugger committed
238
        # For nn.DataParallel compatibility in PyTorch > 1.5
Lysandre Debut's avatar
Lysandre Debut committed
239
240
241
242
243
244
245
246
247
248

        def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
            tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
            return tuples

        gen = parameter._named_members(get_members_fn=find_tensor_attributes)
        first_tuple = next(gen)
        return first_tuple[1].dtype


249
250
def get_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "ModuleUtilsMixin"]):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
251
    Returns the first found floating dtype in parameters if there is one, otherwise returns the last dtype it found.
252
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
253
254
255
256
    last_dtype = None
    for t in parameter.parameters():
        last_dtype = t.dtype
        if t.is_floating_point():
257
258
259
            # Adding fix for https://github.com/pytorch/xla/issues/4152
            # Fixes issue where the model code passes a value that is out of range for XLA_USE_BF16=1
            # and XLA_DOWNCAST_BF16=1 so the conversion would cast it to -inf
260
261
            # NOTE: `is_torch_xla_available()` is checked last as it induces a graph break in torch dynamo
            if XLA_USE_BF16 in ENV_VARS_TRUE_VALUES and is_torch_xla_available():
262
                return torch.bfloat16
263
            if XLA_DOWNCAST_BF16 in ENV_VARS_TRUE_VALUES and is_torch_xla_available():
264
                if t.dtype == torch.float:
265
                    return torch.bfloat16
266
267
                if t.dtype == torch.double:
                    return torch.float32
Sylvain Gugger's avatar
Sylvain Gugger committed
268
            return t.dtype
269

Sylvain Gugger's avatar
Sylvain Gugger committed
270
271
272
    if last_dtype is not None:
        # if no floating dtype was found return whatever the first dtype is
        return last_dtype
273

274
275
276
277
278
279
280
281
282
283
284
285
286
    # For nn.DataParallel compatibility in PyTorch > 1.5
    def find_tensor_attributes(module: nn.Module) -> List[Tuple[str, Tensor]]:
        tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
        return tuples

    gen = parameter._named_members(get_members_fn=find_tensor_attributes)
    last_tuple = None
    for tuple in gen:
        last_tuple = tuple
        if tuple[1].is_floating_point():
            return tuple[1].dtype

    if last_tuple is not None:
287
288
        # fallback to the last dtype
        return last_tuple[1].dtype
289

290
291
292
293
294
295
296
    # fallback to buffer dtype
    for t in parameter.buffers():
        last_dtype = t.dtype
        if t.is_floating_point():
            return t.dtype
    return last_dtype

297
298
299
300
301
302
303
304
305
306
307
308
309
310

def get_state_dict_float_dtype(state_dict):
    """
    Returns the first found floating dtype in `state_dict` or asserts if none were found.
    """
    for t in state_dict.values():
        if t.is_floating_point():
            return t.dtype

    raise ValueError("couldn't find any floating point dtypes in state_dict")


def get_state_dict_dtype(state_dict):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
311
    Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype.
312
313
314
315
316
317
318
    """
    for t in state_dict.values():
        if t.is_floating_point():
            return t.dtype

    # if no floating dtype was found return whatever the first dtype is
    else:
Sylvain Gugger's avatar
Sylvain Gugger committed
319
        return next(state_dict.values()).dtype
320
321


Sylvain Gugger's avatar
Sylvain Gugger committed
322
323
324
325
326
327
328
329
330
331
332
333
334
def dtype_byte_size(dtype):
    """
    Returns the size (in bytes) occupied by one parameter of type `dtype`.

    Example:

    ```py
    >>> dtype_byte_size(torch.float32)
    4
    ```
    """
    if dtype == torch.bool:
        return 1 / 8
335
    bit_search = re.search(r"[^\d](\d+)_?", str(dtype))
Sylvain Gugger's avatar
Sylvain Gugger committed
336
337
338
339
340
341
    if bit_search is None:
        raise ValueError(f"`dtype` is not a valid dtype: {dtype}.")
    bit_size = int(bit_search.groups()[0])
    return bit_size // 8


342
343
344
345
346
347
def check_support_param_buffer_assignment(model_to_load, state_dict, start_prefix=""):
    """
    Checks if `model_to_load` supports param buffer assignment (such
    as when loading in empty weights) by first checking
    if the model explicitly disables it, then by ensuring that the state dict keys
    are a subset of the model's parameters.
348
349

    Note: We fully disable this if we are using `deepspeed`
350
351
352
353
    """
    if len([key for key in state_dict if key.startswith(start_prefix)]) == 0:
        return False

354
355
356
    if is_deepspeed_zero3_enabled():
        return False

357
    # Some models explicitly do not support param buffer assignment
358
    if not getattr(model_to_load, "_supports_param_buffer_assignment", True):
359
360
361
362
363
364
365
366
367
368
369
370
371
372
        logger.debug(
            f"{model_to_load.__class__.__name__} does not support param buffer assignment, loading will be slower"
        )
        return False

    # If the model does, the incoming `state_dict` and the `model_to_load` must be the same dtype
    first_key = list(model_to_load.state_dict().keys())[0]
    if start_prefix + first_key in state_dict:
        return state_dict[start_prefix + first_key].dtype == model_to_load.state_dict()[first_key].dtype

    # For cases when the `state_dict` doesn't contain real weights to the model (`test_model_weights_reload_no_missing_tied_weights`)
    return False


373
374
375
def shard_checkpoint(
    state_dict: Dict[str, torch.Tensor], max_shard_size: Union[int, str] = "10GB", weights_name: str = WEIGHTS_NAME
):
Sylvain Gugger's avatar
Sylvain Gugger committed
376
377
378
379
380
381
382
383
384
385
386
    """
    Splits a model state dictionary in sub-checkpoints so that the final size of each sub-checkpoint does not exceed a
    given size.

    The sub-checkpoints are determined by iterating through the `state_dict` in the order of its keys, so there is no
    optimization made to make each sub-checkpoint as close as possible to the maximum size passed. For example, if the
    limit is 10GB and we have weights of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB],
    [6+2+2GB] and not [6+2+2GB], [6+2GB], [6GB].

    <Tip warning={true}>

Alan Ji's avatar
Alan Ji committed
387
    If one of the model's weight is bigger than `max_shard_size`, it will end up in its own sub-checkpoint which will
Sylvain Gugger's avatar
Sylvain Gugger committed
388
389
390
391
392
393
394
395
396
    have a size greater than `max_shard_size`.

    </Tip>

    Args:
        state_dict (`Dict[str, torch.Tensor]`): The state dictionary of a model to save.
        max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`):
            The maximum size of each sub-checkpoint. If expressed as a string, needs to be digits followed by a unit
            (like `"5MB"`).
397
398
        weights_name (`str`, *optional*, defaults to `"pytorch_model.bin"`):
            The name of the model save file.
Sylvain Gugger's avatar
Sylvain Gugger committed
399
    """
400
401
402
403
    logger.warning(
        "Note that `shard_checkpoint` is deprecated and will be removed in v4.44. We recommend you using "
        "split_torch_state_dict_into_shards from huggingface_hub library"
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
404
405
    max_shard_size = convert_file_size_to_int(max_shard_size)

Thomas Wang's avatar
Thomas Wang committed
406
407
    sharded_state_dicts = [{}]
    last_block_size = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
408
    total_size = 0
Thomas Wang's avatar
Thomas Wang committed
409
    storage_id_to_block = {}
Sylvain Gugger's avatar
Sylvain Gugger committed
410
411

    for key, weight in state_dict.items():
412
413
414
415
416
417
        # when bnb serialization is used the weights in the state dict can be strings
        # check: https://github.com/huggingface/transformers/pull/24416 for more details
        if isinstance(weight, str):
            continue
        else:
            storage_id = id_tensor_storage(weight)
Thomas Wang's avatar
Thomas Wang committed
418
419

        # If a `weight` shares the same underlying storage as another tensor, we put `weight` in the same `block`
420
        if storage_id in storage_id_to_block and weight.device != torch.device("meta"):
Thomas Wang's avatar
Thomas Wang committed
421
422
423
424
            block_id = storage_id_to_block[storage_id]
            sharded_state_dicts[block_id][key] = weight
            continue

Sylvain Gugger's avatar
Sylvain Gugger committed
425
        weight_size = weight.numel() * dtype_byte_size(weight.dtype)
Sylvain Gugger's avatar
Sylvain Gugger committed
426
427
428
        # If this weight is going to tip up over the maximal size, we split, but only if we have put at least one
        # weight in the current shard.
        if last_block_size + weight_size > max_shard_size and len(sharded_state_dicts[-1]) > 0:
Thomas Wang's avatar
Thomas Wang committed
429
430
            sharded_state_dicts.append({})
            last_block_size = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
431

Thomas Wang's avatar
Thomas Wang committed
432
433
        sharded_state_dicts[-1][key] = weight
        last_block_size += weight_size
Sylvain Gugger's avatar
Sylvain Gugger committed
434
        total_size += weight_size
Thomas Wang's avatar
Thomas Wang committed
435
        storage_id_to_block[storage_id] = len(sharded_state_dicts) - 1
Sylvain Gugger's avatar
Sylvain Gugger committed
436
437
438

    # If we only have one shard, we return it
    if len(sharded_state_dicts) == 1:
439
        return {weights_name: sharded_state_dicts[0]}, None
Sylvain Gugger's avatar
Sylvain Gugger committed
440
441
442
443
444

    # Otherwise, let's build the index
    weight_map = {}
    shards = {}
    for idx, shard in enumerate(sharded_state_dicts):
445
446
447
448
        shard_file = weights_name.replace(".bin", f"-{idx+1:05d}-of-{len(sharded_state_dicts):05d}.bin")
        shard_file = shard_file.replace(
            ".safetensors", f"-{idx + 1:05d}-of-{len(sharded_state_dicts):05d}.safetensors"
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
449
450
451
452
453
454
455
456
457
458
        shards[shard_file] = shard
        for key in shard.keys():
            weight_map[key] = shard_file

    # Add the metadata
    metadata = {"total_size": total_size}
    index = {"metadata": metadata, "weight_map": weight_map}
    return shards, index


459
def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True):
460
461
462
463
464
465
466
467
468
469
470
471
472
    """
    This is the same as
    [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict)
    but for a sharded checkpoint.

    This load is performed efficiently: each checkpoint shard is loaded one by one in RAM and deleted after being
    loaded in the model.

    Args:
        model (`torch.nn.Module`): The model in which to load the checkpoint.
        folder (`str` or `os.PathLike`): A path to a folder containing the sharded checkpoint.
        strict (`bool`, *optional`, defaults to `True`):
            Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint.
473
474
475
        prefer_safe (`bool`, *optional*, defaults to `False`)
            If both safetensors and PyTorch save files are present in checkpoint and `prefer_safe` is True, the
            safetensors files will be loaded. Otherwise, PyTorch files are always loaded when possible.
476
477
478
479
480
481
482
483

    Returns:
        `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields
            - `missing_keys` is a list of str containing the missing keys
            - `unexpected_keys` is a list of str containing the unexpected keys
    """
    # Load the index
    index_file = os.path.join(folder, WEIGHTS_INDEX_NAME)
484
    safe_index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME)
485

486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
    index_present = os.path.isfile(index_file)
    safe_index_present = os.path.isfile(safe_index_file)

    if not index_present and not (safe_index_present and is_safetensors_available()):
        filenames = (
            (WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_INDEX_NAME) if is_safetensors_available() else (WEIGHTS_INDEX_NAME,)
        )
        raise ValueError(f"Can't find a checkpoint index ({' or '.join(filenames)}) in {folder}.")

    load_safe = False
    if safe_index_present:
        if prefer_safe:
            if is_safetensors_available():
                load_safe = True  # load safe due to preference
            else:
                logger.warning(
                    f"Cannot load sharded checkpoint at {folder} safely since safetensors is not installed!"
                )
        elif not index_present:
            load_safe = True  # load safe since we have no other choice

    load_index = safe_index_file if load_safe else index_file

    with open(load_index, "r", encoding="utf-8") as f:
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
        index = json.load(f)

    shard_files = list(set(index["weight_map"].values()))

    # If strict=True, error before loading any of the state dicts.
    loaded_keys = index["weight_map"].keys()
    model_keys = model.state_dict().keys()
    missing_keys = [key for key in model_keys if key not in loaded_keys]
    unexpected_keys = [key for key in loaded_keys if key not in model_keys]
    if strict and (len(missing_keys) > 0 or len(unexpected_keys) > 0):
        error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}"
        if len(missing_keys) > 0:
            str_missing_keys = ",".join([f'"{k}"' for k in missing_keys])
            error_message += f"\nMissing key(s): {str_missing_keys}."
        if len(unexpected_keys) > 0:
            str_unexpected_keys = ",".join([f'"{k}"' for k in unexpected_keys])
            error_message += f"\nMissing key(s): {str_unexpected_keys}."
        raise RuntimeError(error_message)

Yih-Dar's avatar
Yih-Dar committed
529
530
    weights_only_kwarg = {"weights_only": True} if is_torch_greater_or_equal_than_1_13 else {}
    loader = safe_load_file if load_safe else partial(torch.load, map_location="cpu", **weights_only_kwarg)
531

532
    for shard_file in shard_files:
533
        state_dict = loader(os.path.join(folder, shard_file))
534
535
        model.load_state_dict(state_dict, strict=False)

536
        # Make sure memory is freed before we load the next state dict.
537
538
539
540
541
542
543
        del state_dict
        gc.collect()

    # Return the same thing as PyTorch load_state_dict function.
    return torch.nn.modules.module._IncompatibleKeys(missing_keys, unexpected_keys)


544
def load_state_dict(checkpoint_file: Union[str, os.PathLike], is_quantized: bool = False):
Sylvain Gugger's avatar
Sylvain Gugger committed
545
546
547
    """
    Reads a PyTorch checkpoint file, returning properly formatted errors if they arise.
    """
548
549
550
551
    if checkpoint_file.endswith(".safetensors") and is_safetensors_available():
        # Check format of the archive
        with safe_open(checkpoint_file, framework="pt") as f:
            metadata = f.metadata()
552
        if metadata.get("format") not in ["pt", "tf", "flax", "mlx"]:
553
554
555
556
557
            raise OSError(
                f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure "
                "you save your model with the `save_pretrained` method."
            )
        return safe_load_file(checkpoint_file)
Sylvain Gugger's avatar
Sylvain Gugger committed
558
    try:
559
        if (
560
561
562
            (is_deepspeed_zero3_enabled() and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0)
            or (is_fsdp_enabled() and not is_local_dist_rank_0())
        ) and not is_quantized:
563
564
565
            map_location = "meta"
        else:
            map_location = "cpu"
566
567
568
569
570
571
572
573
574
        extra_args = {}
        # mmap can only be used with files serialized with zipfile-based format.
        if (
            isinstance(checkpoint_file, str)
            and map_location != "meta"
            and version.parse(torch.__version__) >= version.parse("2.1.0")
            and is_zipfile(checkpoint_file)
        ):
            extra_args = {"mmap": True}
Yih-Dar's avatar
Yih-Dar committed
575
        weights_only_kwarg = {"weights_only": True} if is_torch_greater_or_equal_than_1_13 else {}
576
577
578
        return torch.load(
            checkpoint_file,
            map_location=map_location,
Yih-Dar's avatar
Yih-Dar committed
579
            **weights_only_kwarg,
580
581
            **extra_args,
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
582
583
584
    except Exception as e:
        try:
            with open(checkpoint_file) as f:
585
                if f.read(7) == "version":
Sylvain Gugger's avatar
Sylvain Gugger committed
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
                    raise OSError(
                        "You seem to have cloned a repository without having git-lfs installed. Please install "
                        "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder "
                        "you cloned."
                    )
                else:
                    raise ValueError(
                        f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained "
                        "model. Make sure you have saved the model properly."
                    ) from e
        except (UnicodeDecodeError, ValueError):
            raise OSError(
                f"Unable to load weights from pytorch checkpoint file for '{checkpoint_file}' "
                f"at '{checkpoint_file}'. "
                "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True."
            )


604
605
606
607
608
def set_initialized_submodules(model, state_dict_keys):
    """
    Sets the `_is_hf_initialized` flag in all submodules of a given model when all its weights are in the loaded state
    dict.
    """
609
    not_initialized_submodules = {}
610
    for module_name, module in model.named_modules():
611
612
        loaded_keys = {k.replace(f"{module_name}.", "") for k in state_dict_keys if k.startswith(f"{module_name}.")}
        if loaded_keys.issuperset(module.state_dict()):
613
            module._is_hf_initialized = True
614
615
616
        else:
            not_initialized_submodules[module_name] = module
    return not_initialized_submodules
617
618


619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
def _end_ptr(tensor: torch.Tensor) -> int:
    # extract the end of the pointer if the tensor is a slice of a bigger tensor
    if tensor.nelement():
        stop = tensor.view(-1)[-1].data_ptr() + tensor.element_size()
    else:
        stop = tensor.data_ptr()
    return stop


def _get_tied_weight_keys(module: nn.Module, prefix=""):
    tied_weight_keys = []
    if getattr(module, "_tied_weights_keys", None) is not None:
        names = [f"{prefix}.{k}" if prefix else k for k in module._tied_weights_keys]
        tied_weight_keys.extend(names)
    if getattr(module, "_dynamic_tied_weights_keys", None) is not None:
        names = [f"{prefix}.{k}" if prefix else k for k in module._dynamic_tied_weights_keys]
        tied_weight_keys.extend(names)
    for name, submodule in module.named_children():
        local_prefix = f"{prefix}.{name}" if prefix else name
        tied_weight_keys.extend(_get_tied_weight_keys(submodule, prefix=local_prefix))
    return tied_weight_keys


def _find_disjoint(tensors: List[Set[str]], state_dict: Dict[str, torch.Tensor]) -> Tuple[List[Set[str]], List[str]]:
    filtered_tensors = []
    for shared in tensors:
        if len(shared) < 2:
            filtered_tensors.append(shared)
            continue

        areas = []
        for name in shared:
            tensor = state_dict[name]
            areas.append((tensor.data_ptr(), _end_ptr(tensor), name))
        areas.sort()

        _, last_stop, last_name = areas[0]
        filtered_tensors.append({last_name})
        for start, stop, name in areas[1:]:
            if start >= last_stop:
                filtered_tensors.append({name})
            else:
                filtered_tensors[-1].add(name)
            last_stop = stop
    disjoint_tensors = []
    shared_tensors = []
    for tensors in filtered_tensors:
        if len(tensors) == 1:
            disjoint_tensors.append(tensors.pop())
        else:
            shared_tensors.append(tensors)
    return shared_tensors, disjoint_tensors


def _find_identical(tensors: List[Set[str]], state_dict: Dict[str, torch.Tensor]) -> Tuple[List[Set[str]], Set[str]]:
    shared_tensors = []
    identical = []
    for shared in tensors:
        if len(shared) < 2:
            continue

        areas = collections.defaultdict(set)
        for name in shared:
            tensor = state_dict[name]
            area = (tensor.device, tensor.data_ptr(), _end_ptr(tensor))
            areas[area].add(name)
        if len(areas) == 1:
            identical.append(shared)
        else:
            shared_tensors.append(shared)
    return shared_tensors, identical


692
def _load_state_dict_into_model(model_to_load, state_dict, start_prefix, assign_to_params_buffers=False):
Sylvain Gugger's avatar
Sylvain Gugger committed
693
694
695
696
697
698
    # Convert old format to new format if needed from a PyTorch state_dict
    old_keys = []
    new_keys = []
    for key in state_dict.keys():
        new_key = None
        if "gamma" in key:
699
            logger.warning(PARAM_RENAME_WARNING.format("gamma", "weight"))
Sylvain Gugger's avatar
Sylvain Gugger committed
700
701
            new_key = key.replace("gamma", "weight")
        if "beta" in key:
702
            logger.warning(PARAM_RENAME_WARNING.format("beta", "bias"))
Sylvain Gugger's avatar
Sylvain Gugger committed
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
            new_key = key.replace("beta", "bias")
        if new_key:
            old_keys.append(key)
            new_keys.append(new_key)
    for old_key, new_key in zip(old_keys, new_keys):
        state_dict[new_key] = state_dict.pop(old_key)

    # copy state_dict so _load_from_state_dict can modify it
    metadata = getattr(state_dict, "_metadata", None)
    state_dict = state_dict.copy()
    if metadata is not None:
        state_dict._metadata = metadata

    error_msgs = []

    # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants
    # so we need to apply the function recursively.
720
    def load(module: nn.Module, state_dict, prefix="", assign_to_params_buffers=False):
Sylvain Gugger's avatar
Sylvain Gugger committed
721
        local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
722
723
        local_metadata["assign_to_params_buffers"] = assign_to_params_buffers

724
        args = (state_dict, prefix, local_metadata, True, [], [], error_msgs)
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
        # Parameters of module and children will start with prefix. We can exit early if there are none in this
        # state_dict
        if len([key for key in state_dict if key.startswith(prefix)]) > 0:
            if is_deepspeed_zero3_enabled():
                import deepspeed

                # In sharded models, each shard has only part of the full state_dict, so only gather
                # parameters that are in the current state_dict.
                named_parameters = dict(module.named_parameters(prefix=prefix[:-1], recurse=False))
                params_to_gather = [named_parameters[k] for k in state_dict.keys() if k in named_parameters]
                if len(params_to_gather) > 0:
                    # because zero3 puts placeholders in model params, this context
                    # manager gathers (unpartitions) the params of the current layer, then loads from
                    # the state dict and then re-partitions them again
                    with deepspeed.zero.GatheredParameters(params_to_gather, modifier_rank=0):
                        if torch.distributed.get_rank() == 0:
                            module._load_from_state_dict(*args)
            else:
                module._load_from_state_dict(*args)
Sylvain Gugger's avatar
Sylvain Gugger committed
744
745
746

        for name, child in module._modules.items():
            if child is not None:
747
                load(child, state_dict, prefix + name + ".", assign_to_params_buffers)
Sylvain Gugger's avatar
Sylvain Gugger committed
748

749
    load(model_to_load, state_dict, prefix=start_prefix, assign_to_params_buffers=assign_to_params_buffers)
750
751
752
    # Delete `state_dict` so it could be collected by GC earlier. Note that `state_dict` is a copy of the argument, so
    # it's safe to delete it.
    del state_dict
Sylvain Gugger's avatar
Sylvain Gugger committed
753
754
755
756

    return error_msgs


757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
def find_submodule_and_param_name(model, long_key, start_prefix):
    """
    A helper util to find the last sub-module and the param/buffer name. If `start_prefix` is supplied it'll be removed
    from the start of the key
    """

    if len(start_prefix) > 0 and long_key.startswith(start_prefix):
        long_key = ".".join(long_key.split(".")[1:])

    split_key = long_key.split(".")
    submodule = model
    while len(split_key) > 1:
        if hasattr(submodule, split_key[0]):
            submodule = getattr(submodule, split_key[0])
            del split_key[0]
        else:
            submodule = None
            break
    if submodule == model:
        submodule = None
    return submodule, split_key[0]


def _move_model_to_meta(model, loaded_state_dict_keys, start_prefix):
    """
    Moves `loaded_state_dict_keys` in model to meta device which frees up the memory taken by those params.

    `start_prefix` is used for models which insert their name into model keys, e.g. `bert` in
    `bert.pooler.dense.weight`

    """

    # dematerialize param storage for keys that are going to be replaced by state_dict, by
    # putting those on the meta device
    for k in loaded_state_dict_keys:
        submodule, param_name = find_submodule_and_param_name(model, k, start_prefix)
        if submodule is not None:
            # selectively switch to the meta device only those params/buffers that will
            # be next replaced from state_dict. This a complex way to do p.to_("meta")
            # since we have no in-place to_ for tensors.
            new_val = getattr(submodule, param_name)
            if isinstance(new_val, torch.nn.Parameter):
                # isinstance returns False for Params on meta device, so switch after the check
                new_val = torch.nn.Parameter(new_val.to("meta"))
            else:
                new_val = new_val.to("meta")
            setattr(submodule, param_name, new_val)


806
807
808
809
810
811
812
813
814
815
816
817
def _load_state_dict_into_meta_model(
    model,
    state_dict,
    loaded_state_dict_keys,  # left for now but could be removed, see below
    start_prefix,
    expected_keys,
    device_map=None,
    offload_folder=None,
    offload_index=None,
    state_dict_folder=None,
    state_dict_index=None,
    dtype=None,
818
    hf_quantizer=None,
Sylvain Gugger's avatar
Sylvain Gugger committed
819
    is_safetensors=False,
820
    keep_in_fp32_modules=None,
821
    unexpected_keys=None,  # passing `unexpected` for cleanup from quantization items
822
):
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
    """
    This is somewhat similar to `_load_state_dict_into_model`, but deals with a model that has some or all of its
    params on a `meta` device. It replaces the model params with the data from the `state_dict`, while moving the
    params back to the normal device, but only for `loaded_state_dict_keys`.

    `start_prefix` is used for models which insert their name into model keys, e.g. `bert` in
    `bert.pooler.dense.weight`

    """

    # XXX: remaining features to implement to be fully compatible with _load_state_dict_into_model
    # - deepspeed zero 3 support
    # - need to copy metadata if any - see _load_state_dict_into_model
    # - handling error_msgs - mimicking the error handling in module._load_from_state_dict()
    # - Is there a situation where some keys aren't in `loaded_state_dict_keys` and in which case
    #   they won't get loaded.

    error_msgs = []

842
843
    old_keys = []
    new_keys = []
844
    is_quantized = hf_quantizer is not None
845
846
847
    for key in state_dict.keys():
        new_key = None
        if "gamma" in key:
848
            logger.warning(PARAM_RENAME_WARNING.format("gamma", "weight"))
849
850
            new_key = key.replace("gamma", "weight")
        if "beta" in key:
851
            logger.warning(PARAM_RENAME_WARNING.format("beta", "bias"))
852
853
854
855
856
857
            new_key = key.replace("beta", "bias")
        if new_key:
            old_keys.append(key)
            new_keys.append(new_key)
    for old_key, new_key in zip(old_keys, new_keys):
        state_dict[new_key] = state_dict.pop(old_key)
858

859
860
    is_torch_e4m3fn_available = hasattr(torch, "float8_e4m3fn")

861
862
863
864
865
866
867
868
869
    for param_name, param in state_dict.items():
        # First part of the test is always true as load_state_dict_keys always contains state_dict keys.
        if param_name not in loaded_state_dict_keys or param_name not in expected_keys:
            continue

        if param_name.startswith(start_prefix):
            param_name = param_name[len(start_prefix) :]

        module_name = param_name
870
        set_module_kwargs = {}
871

872
        # We convert floating dtypes to the `dtype` passed except for float8_e4m3fn type. We also want to keep the buffers/params
873
        # in int/uint/bool and not cast them.
874
875
        is_param_float8_e4m3fn = is_torch_e4m3fn_available and param.dtype == torch.float8_e4m3fn
        if dtype is not None and torch.is_floating_point(param) and not is_param_float8_e4m3fn:
876
877
            if (
                keep_in_fp32_modules is not None
878
879
880
                and any(
                    module_to_keep_in_fp32 in param_name.split(".") for module_to_keep_in_fp32 in keep_in_fp32_modules
                )
881
882
883
                and dtype == torch.float16
            ):
                param = param.to(torch.float32)
884
885
886
887
888

                # For backward compatibility with older versions of `accelerate`
                # TODO: @sgugger replace this check with version check at the next `accelerate` release
                if "dtype" in list(inspect.signature(set_module_tensor_to_device).parameters):
                    set_module_kwargs["dtype"] = torch.float32
889
890
            else:
                param = param.to(dtype)
891

892
893
894
895
896
897
898
899
900
901
902
        # For compatibility with PyTorch load_state_dict which converts state dict dtype to existing dtype in model, and which
        # uses `param.copy_(input_param)` that preserves the contiguity of the parameter in the model.
        # Reference: https://github.com/pytorch/pytorch/blob/db79ceb110f6646523019a59bbd7b838f43d4a86/torch/nn/modules/module.py#L2040C29-L2040C29
        old_param = model
        splits = param_name.split(".")
        for split in splits:
            old_param = getattr(old_param, split)
            if old_param is None:
                break
        if old_param is not None:
            if dtype is None:
903
                param = param.to(old_param.dtype)
904

905
906
907
            if old_param.is_contiguous():
                param = param.contiguous()

908
909
        set_module_kwargs["value"] = param

910
911
912
913
914
915
916
917
918
919
920
        if device_map is None:
            param_device = "cpu"
        else:
            # find next higher level module that is defined in device_map:
            # bert.lm_head.weight -> bert.lm_head -> bert -> ''
            while len(module_name) > 0 and module_name not in device_map:
                module_name = ".".join(module_name.split(".")[:-1])
            if module_name == "" and "" not in device_map:
                # TODO: group all errors and raise at the end.
                raise ValueError(f"{param_name} doesn't have any device set.")
            param_device = device_map[module_name]
921

922
        if param_device == "disk":
Sylvain Gugger's avatar
Sylvain Gugger committed
923
924
            if not is_safetensors:
                offload_index = offload_weight(param, param_name, offload_folder, offload_index)
925
        elif param_device == "cpu" and state_dict_index is not None:
926
            state_dict_index = offload_weight(param, param_name, state_dict_folder, state_dict_index)
927
        elif (
928
            not is_quantized
929
            or (not hf_quantizer.requires_parameters_quantization)
930
931
932
933
934
            or (
                not hf_quantizer.check_quantized_param(
                    model, param, param_name, state_dict, param_device=param_device, device_map=device_map
                )
            )
935
936
        ):
            # For backward compatibility with older versions of `accelerate` and for non-quantized params
937
            set_module_tensor_to_device(model, param_name, param_device, **set_module_kwargs)
938
        else:
939
            hf_quantizer.create_quantized_param(model, param, param_name, param_device, state_dict, unexpected_keys)
940
941
942
943
944
945
            # For quantized modules with FSDP/DeepSpeed Stage 3, we need to quantize the parameter on the GPU
            # and then cast it to CPU to avoid excessive memory usage on each GPU
            # in comparison to the sharded model across GPUs.
            if is_fsdp_enabled() or is_deepspeed_zero3_enabled():
                module, tensor_name = get_module_from_name(model, param_name)
                value = getattr(module, tensor_name)
946
                value = type(value)(value.data.to("cpu"), **value.__dict__)
947
                setattr(module, tensor_name, value)
948
            # TODO: consider removing used param_parts from state_dict before return
949
950

    return error_msgs, offload_index, state_dict_index
951
952


953
954
955
956
957
958
959
960
961
def _add_variant(weights_name: str, variant: Optional[str] = None) -> str:
    if variant is not None:
        splits = weights_name.split(".")
        splits = splits[:-1] + [variant] + splits[-1:]
        weights_name = ".".join(splits)

    return weights_name


962
class ModuleUtilsMixin:
Julien Chaumond's avatar
Julien Chaumond committed
963
    """
964
    A few utilities for `torch.nn.Modules`, to be used as a mixin.
Julien Chaumond's avatar
Julien Chaumond committed
965
966
    """

967
968
969
970
    @staticmethod
    def _hook_rss_memory_pre_forward(module, *args, **kwargs):
        try:
            import psutil
Sylvain Gugger's avatar
Sylvain Gugger committed
971
        except ImportError:
972
973
974
975
976
977
978
979
980
981
982
            raise ImportError("You need to install psutil (pip install psutil) to use memory tracing.")

        process = psutil.Process(os.getpid())
        mem = process.memory_info()
        module.mem_rss_pre_forward = mem.rss
        return None

    @staticmethod
    def _hook_rss_memory_post_forward(module, *args, **kwargs):
        try:
            import psutil
Sylvain Gugger's avatar
Sylvain Gugger committed
983
        except ImportError:
984
985
986
987
988
989
990
991
992
993
            raise ImportError("You need to install psutil (pip install psutil) to use memory tracing.")

        process = psutil.Process(os.getpid())
        mem = process.memory_info()
        module.mem_rss_post_forward = mem.rss
        mem_rss_diff = module.mem_rss_post_forward - module.mem_rss_pre_forward
        module.mem_rss_diff = mem_rss_diff + (module.mem_rss_diff if hasattr(module, "mem_rss_diff") else 0)
        return None

    def add_memory_hooks(self):
Sylvain Gugger's avatar
Sylvain Gugger committed
994
995
996
        """
        Add a memory hook before and after each sub-module forward pass to record increase in memory consumption.

Sylvain Gugger's avatar
Sylvain Gugger committed
997
998
        Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to zero
        with `model.reset_memory_hooks_state()`.
999
1000
1001
1002
1003
1004
1005
        """
        for module in self.modules():
            module.register_forward_pre_hook(self._hook_rss_memory_pre_forward)
            module.register_forward_hook(self._hook_rss_memory_post_forward)
        self.reset_memory_hooks_state()

    def reset_memory_hooks_state(self):
Sylvain Gugger's avatar
Sylvain Gugger committed
1006
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
1007
        Reset the `mem_rss_diff` attribute of each module (see [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]).
Sylvain Gugger's avatar
Sylvain Gugger committed
1008
        """
1009
1010
1011
1012
1013
        for module in self.modules():
            module.mem_rss_diff = 0
            module.mem_rss_post_forward = 0
            module.mem_rss_pre_forward = 0

1014
    @property
Sylvain Gugger's avatar
Sylvain Gugger committed
1015
    def device(self) -> torch.device:
1016
        """
1017
        `torch.device`: The device on which the module is (assuming that all the module parameters are on the same
1018
        device).
1019
        """
Lysandre Debut's avatar
Lysandre Debut committed
1020
        return get_parameter_device(self)
1021

1022
    @property
1023
    def dtype(self) -> torch.dtype:
1024
        """
1025
        `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
1026
        """
Lysandre Debut's avatar
Lysandre Debut committed
1027
        return get_parameter_dtype(self)
1028
1029

    def invert_attention_mask(self, encoder_attention_mask: Tensor) -> Tensor:
Sylvain Gugger's avatar
Sylvain Gugger committed
1030
1031
1032
1033
        """
        Invert an attention mask (e.g., switches 0. and 1.).

        Args:
1034
            encoder_attention_mask (`torch.Tensor`): An attention mask.
Sylvain Gugger's avatar
Sylvain Gugger committed
1035
1036

        Returns:
1037
            `torch.Tensor`: The inverted attention mask.
Sylvain Gugger's avatar
Sylvain Gugger committed
1038
        """
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
        if encoder_attention_mask.dim() == 3:
            encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
        if encoder_attention_mask.dim() == 2:
            encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
        # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
        # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
        # /transformer/transformer_layers.py#L270
        # encoder_extended_attention_mask = (encoder_extended_attention_mask ==
        # encoder_extended_attention_mask.transpose(-1, -2))
        encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
Yih-Dar's avatar
Yih-Dar committed
1049
        encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * torch.finfo(self.dtype).min
1050

1051
1052
        return encoder_extended_attention_mask

1053
    @staticmethod
1054
1055
1056
1057
1058
1059
1060
    def create_extended_attention_mask_for_decoder(input_shape, attention_mask, device=None):
        if device is not None:
            warnings.warn(
                "The `device` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
            )
        else:
            device = attention_mask.device
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
        batch_size, seq_length = input_shape
        seq_ids = torch.arange(seq_length, device=device)
        causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
        # in case past_key_values are used we need to add a prefix ones mask to the causal mask
        # causal and attention masks must have same type with pytorch version < 1.3
        causal_mask = causal_mask.to(attention_mask.dtype)

        if causal_mask.shape[1] < attention_mask.shape[1]:
            prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
            causal_mask = torch.cat(
                [
                    torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
                    causal_mask,
                ],
                axis=-1,
            )

        extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
        return extended_attention_mask

1081
    def get_extended_attention_mask(
1082
        self, attention_mask: Tensor, input_shape: Tuple[int], device: torch.device = None, dtype: torch.float = None
1083
    ) -> Tensor:
Sylvain Gugger's avatar
Sylvain Gugger committed
1084
1085
        """
        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
1086
1087

        Arguments:
1088
            attention_mask (`torch.Tensor`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1089
                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
1090
            input_shape (`Tuple[int]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1091
                The shape of the input to the model.
1092
1093

        Returns:
1094
            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
1095
        """
Yih-Dar's avatar
Yih-Dar committed
1096
1097
1098
        if dtype is None:
            dtype = self.dtype

1099
1100
1101
1102
1103
1104
        if not (attention_mask.dim() == 2 and self.config.is_decoder):
            # show warning only if it won't be shown in `create_extended_attention_mask_for_decoder`
            if device is not None:
                warnings.warn(
                    "The `device` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning
                )
1105
1106
1107
1108
1109
1110
1111
1112
1113
        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        if attention_mask.dim() == 3:
            extended_attention_mask = attention_mask[:, None, :, :]
        elif attention_mask.dim() == 2:
            # Provided a padding mask of dimensions [batch_size, seq_length]
            # - if the model is a decoder, apply a causal mask in addition to the padding mask
            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
            if self.config.is_decoder:
1114
                extended_attention_mask = ModuleUtilsMixin.create_extended_attention_mask_for_decoder(
1115
1116
                    input_shape, attention_mask, device
                )
1117
1118
1119
1120
            else:
                extended_attention_mask = attention_mask[:, None, None, :]
        else:
            raise ValueError(
1121
                f"Wrong shape for input_ids (shape {input_shape}) or attention_mask (shape {attention_mask.shape})"
1122
1123
1124
1125
            )

        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
        # masked positions, this operation will create a tensor which is 0.0 for
1126
        # positions we want to attend and the dtype's smallest value for masked positions.
1127
1128
        # Since we are adding it to the raw scores before the softmax, this is
        # effectively the same as removing these entirely.
Yih-Dar's avatar
Yih-Dar committed
1129
1130
        extended_attention_mask = extended_attention_mask.to(dtype=dtype)  # fp16 compatibility
        extended_attention_mask = (1.0 - extended_attention_mask) * torch.finfo(dtype).min
1131
1132
        return extended_attention_mask

Sylvain Gugger's avatar
Sylvain Gugger committed
1133
1134
1135
    def get_head_mask(
        self, head_mask: Optional[Tensor], num_hidden_layers: int, is_attention_chunked: bool = False
    ) -> Tensor:
1136
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
1137
1138
1139
        Prepare the head mask if needed.

        Args:
1140
            head_mask (`torch.Tensor` with shape `[num_heads]` or `[num_hidden_layers x num_heads]`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
1141
                The mask indicating if we should keep the heads or not (1.0 for keep, 0.0 for discard).
1142
            num_hidden_layers (`int`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1143
                The number of hidden layers in the model.
1144
            is_attention_chunked (`bool`, *optional*, defaults to `False`):
Sylvain Gugger's avatar
Sylvain Gugger committed
1145
1146
                Whether or not the attentions scores are computed by chunks or not.

1147
        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
1148
1149
            `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or list with
            `[None]` for each layer.
1150
1151
1152
        """
        if head_mask is not None:
            head_mask = self._convert_head_mask_to_5d(head_mask, num_hidden_layers)
Patrick von Platen's avatar
Patrick von Platen committed
1153
1154
            if is_attention_chunked is True:
                head_mask = head_mask.unsqueeze(-1)
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
        else:
            head_mask = [None] * num_hidden_layers

        return head_mask

    def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers):
        """-> [num_hidden_layers x batch x num_heads x seq_length x seq_length]"""
        if head_mask.dim() == 1:
            head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
            head_mask = head_mask.expand(num_hidden_layers, -1, -1, -1, -1)
        elif head_mask.dim() == 2:
            head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)  # We can specify head_mask for each layer
        assert head_mask.dim() == 5, f"head_mask.dim != 5, instead {head_mask.dim()}"
1168
        head_mask = head_mask.to(dtype=self.dtype)  # switch to float if need + fp16 compatibility
1169
1170
        return head_mask

1171
1172
1173
1174
1175
    def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int:
        """
        Get number of (optionally, trainable or non-embeddings) parameters in the module.

        Args:
1176
            only_trainable (`bool`, *optional*, defaults to `False`):
1177
1178
                Whether or not to return only the number of trainable parameters

1179
            exclude_embeddings (`bool`, *optional*, defaults to `False`):
1180
1181
1182
                Whether or not to return only the number of non-embeddings parameters

        Returns:
1183
            `int`: The number of parameters.
1184
1185
        """

1186
1187
1188
1189
        if exclude_embeddings:
            embedding_param_names = [
                f"{name}.weight" for name, module_type in self.named_modules() if isinstance(module_type, nn.Embedding)
            ]
1190
            total_parameters = [
1191
1192
1193
                parameter for name, parameter in self.named_parameters() if name not in embedding_param_names
            ]
        else:
1194
1195
1196
1197
            total_parameters = list(self.parameters())

        total_numel = []
        is_loaded_in_4bit = getattr(self, "is_loaded_in_4bit", False)
1198

1199
1200
1201
1202
1203
1204
        if is_loaded_in_4bit:
            if is_bitsandbytes_available():
                import bitsandbytes as bnb
            else:
                raise ValueError(
                    "bitsandbytes is not installed but it seems that the model has been loaded in 4bit precision, something went wrong"
1205
                    " make sure to install bitsandbytes with `pip install bitsandbytes`. You also need a GPU. "
1206
1207
1208
1209
1210
1211
1212
                )

        for param in total_parameters:
            if param.requires_grad or not only_trainable:
                # For 4bit models, we need to multiply the number of parameters by 2 as half of the parameters are
                # used for the 4bit quantization (uint8 tensors are stored)
                if is_loaded_in_4bit and isinstance(param, bnb.nn.Params4bit):
1213
1214
1215
1216
1217
1218
1219
                    if hasattr(param, "element_size"):
                        num_bytes = param.element_size()
                    elif hasattr(param, "quant_storage"):
                        num_bytes = param.quant_storage.itemsize
                    else:
                        num_bytes = 1
                    total_numel.append(param.numel() * 2 * num_bytes)
1220
1221
1222
1223
                else:
                    total_numel.append(param.numel())

        return sum(total_numel)
1224
1225
1226
1227
1228
1229

    def estimate_tokens(self, input_dict: Dict[str, Union[torch.Tensor, Any]]) -> int:
        """
        Helper function to estimate the total number of tokens from the model inputs.

        Args:
1230
            inputs (`dict`): The model inputs.
1231
1232

        Returns:
1233
            `int`: The total number of tokens.
1234
        """
1235
1236
        if not hasattr(self, "warnings_issued"):
            self.warnings_issued = {}
1237
1238
        if self.main_input_name in input_dict:
            return input_dict[self.main_input_name].numel()
1239
        elif "estimate_tokens" not in self.warnings_issued:
1240
            logger.warning(
1241
1242
                "Could not estimate the number of tokens of the input, floating-point operations will not be computed"
            )
1243
1244
            self.warnings_issued["estimate_tokens"] = True
        return 0
1245
1246
1247
1248
1249
1250
1251

    def floating_point_ops(
        self, input_dict: Dict[str, Union[torch.Tensor, Any]], exclude_embeddings: bool = True
    ) -> int:
        """
        Get number of (optionally, non-embeddings) floating-point operations for the forward and backward passes of a
        batch with this transformer model. Default approximation neglects the quadratic dependency on the number of
Sylvain Gugger's avatar
Sylvain Gugger committed
1252
1253
        tokens (valid if `12 * d_model << sequence_length`) as laid out in [this
        paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter
Sylvain Gugger's avatar
Sylvain Gugger committed
1254
        re-use e.g. Albert or Universal Transformers, or if doing long-range modeling with very high sequence lengths.
1255
1256

        Args:
1257
            batch_size (`int`):
1258
1259
                The batch size for the forward pass.

1260
            sequence_length (`int`):
1261
1262
                The number of tokens in each line of the batch.

1263
            exclude_embeddings (`bool`, *optional*, defaults to `True`):
1264
1265
1266
                Whether or not to count embedding and softmax operations.

        Returns:
1267
            `int`: The number of floating-point operations.
1268
1269
1270
1271
        """

        return 6 * self.estimate_tokens(input_dict) * self.num_parameters(exclude_embeddings=exclude_embeddings)

Julien Chaumond's avatar
Julien Chaumond committed
1272

1273
class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMixin, PeftAdapterMixin):
1274
1275
    r"""
    Base class for all models.
1276

Sylvain Gugger's avatar
Sylvain Gugger committed
1277
1278
    [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading,
    downloading and saving models as well as a few methods common to all models to:
1279

1280
1281
        - resize the input embeddings,
        - prune heads in the self-attention heads.
1282

1283
    Class attributes (overridden by derived classes):
Sylvain Gugger's avatar
Sylvain Gugger committed
1284

Sylvain Gugger's avatar
Sylvain Gugger committed
1285
1286
1287
1288
        - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class
          for this model architecture.
        - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch model,
          taking as arguments:
1289

Sylvain Gugger's avatar
Sylvain Gugger committed
1290
1291
            - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the TensorFlow checkpoint.
            - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to the model.
1292
            - **path** (`str`) -- A path to the TensorFlow checkpoint.
1293

Sylvain Gugger's avatar
Sylvain Gugger committed
1294
1295
        - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived
          classes of the same architecture adding modules on top of the base model.
1296
        - **is_parallelizable** (`bool`) -- A flag indicating whether this model supports model parallelization.
Sylvain Gugger's avatar
Sylvain Gugger committed
1297
1298
        - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP
          models, `pixel_values` for vision models and `input_values` for speech models).
1299
    """
1300

1301
    config_class = None
1302
    base_model_prefix = ""
1303
    main_input_name = "input_ids"
1304
1305
    model_tags = None

1306
    _auto_class = None
1307
    _no_split_modules = None
1308
    _skip_keys_device_placement = None
1309
    _keep_in_fp32_modules = None
1310

1311
1312
    # a list of `re` patterns of `state_dict` keys that should be removed from the list of missing
    # keys we find (keys inside the model but not in the checkpoint) and avoid unnecessary warnings.
1313
    _keys_to_ignore_on_load_missing = None
1314
1315
1316
    # a list of `re` patterns of `state_dict` keys that should be removed from the list of
    # unexpected keys we find (keys inside the checkpoint but not the model) and avoid unnecessary
    # warnings.
1317
    _keys_to_ignore_on_load_unexpected = None
1318
1319
    # a list of `state_dict` keys to ignore when saving the model (useful for keys that aren't
    # trained, but which are either deterministic or tied variables)
1320
    _keys_to_ignore_on_save = None
Sylvain Gugger's avatar
Sylvain Gugger committed
1321
1322
    # a list of `state_dict` keys that are potentially tied to another key in the state_dict.
    _tied_weights_keys = None
1323

1324
    is_parallelizable = False
1325
    supports_gradient_checkpointing = False
1326
    _is_stateful = False
1327

1328
1329
1330
    # Flash Attention 2 support
    _supports_flash_attn_2 = False

1331
1332
1333
    # SDPA support
    _supports_sdpa = False

1334
    # Has support for a `Cache` instance as `past_key_values`? Does it support a `StaticCache`?
1335
    _supports_cache_class = False
1336
    _supports_static_cache = False
1337

1338
1339
1340
    # Has support for a `QuantoQuantizedCache` instance as `past_key_values`
    _supports_quantized_cache = False

1341
    @property
1342
    def dummy_inputs(self) -> Dict[str, torch.Tensor]:
1343
        """
1344
        `Dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network.
1345
        """
1346
        return {"input_ids": torch.tensor(DUMMY_INPUTS)}
1347

1348
1349
1350
1351
1352
1353
1354
    @property
    def framework(self) -> str:
        """
        :str: Identifies that this is a PyTorch model.
        """
        return "pt"

1355
    def __init__(self, config: PretrainedConfig, *inputs, **kwargs):
Julien Chaumond's avatar
Julien Chaumond committed
1356
        super().__init__()
1357
1358
        if not isinstance(config, PretrainedConfig):
            raise ValueError(
1359
1360
1361
                f"Parameter config in `{self.__class__.__name__}(config)` should be an instance of class "
                "`PretrainedConfig`. To create a model from a pretrained model use "
                f"`model = {self.__class__.__name__}.from_pretrained(PRETRAINED_MODEL_NAME)`"
1362
            )
1363
        # Save config and origin of the pretrained weights if given in model
1364
1365
1366
        config = self._autoset_attn_implementation(
            config, torch_dtype=torch.get_default_dtype(), check_device_map=False
        )
1367
        self.config = config
1368

1369
        self.name_or_path = config.name_or_path
1370
        self.warnings_issued = {}
1371
        self.generation_config = GenerationConfig.from_model_config(config) if self.can_generate() else None
1372
1373
1374
1375
        # Overwrite the class attribute to make it an instance attribute, so models like
        # `InstructBlipForConditionalGeneration` can dynamically update it without modifying the class attribute
        # when a different component (e.g. language_model) is used.
        self._keep_in_fp32_modules = copy.copy(self.__class__._keep_in_fp32_modules)
1376
1377
1378
1379
1380
1381
1382
1383
1384

    def post_init(self):
        """
        A method executed at the end of each Transformer model initialization, to execute code that needs the model's
        modules properly initialized (such as weight initialization).
        """
        self.init_weights()
        self._backward_compatibility_gradient_checkpointing()

1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
    def dequantize(self):
        """
        Potentially dequantize the model in case it has been quantized by a quantization method that support
        dequantization.
        """
        hf_quantizer = getattr(self, "hf_quantizer", None)

        if hf_quantizer is None:
            raise ValueError("You need to first quantize your model in order to dequantize it")

        return hf_quantizer.dequantize(self)

1397
1398
1399
1400
1401
    def _backward_compatibility_gradient_checkpointing(self):
        if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False):
            self.gradient_checkpointing_enable()
            # Remove the attribute now that is has been consumed, so it's no saved in the config.
            delattr(self.config, "gradient_checkpointing")
1402

1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
    def add_model_tags(self, tags: Union[List[str], str]) -> None:
        r"""
        Add custom tags into the model that gets pushed to the Hugging Face Hub. Will
        not overwrite existing tags in the model.

        Args:
            tags (`Union[List[str], str]`):
                The desired tags to inject in the model

        Examples:

        ```python
        from transformers import AutoModel

1417
        model = AutoModel.from_pretrained("google-bert/bert-base-cased")
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434

        model.add_model_tags(["custom", "custom-bert"])

        # Push the model to your namespace with the name "my-custom-bert".
        model.push_to_hub("my-custom-bert")
        ```
        """
        if isinstance(tags, str):
            tags = [tags]

        if self.model_tags is None:
            self.model_tags = []

        for tag in tags:
            if tag not in self.model_tags:
                self.model_tags.append(tag)

1435
1436
1437
1438
1439
1440
    @classmethod
    def _from_config(cls, config, **kwargs):
        """
        All context managers that the model should be initialized under go here.

        Args:
1441
1442
            torch_dtype (`torch.dtype`, *optional*):
                Override the default `torch.dtype` and load the model under this dtype.
1443
1444
        """
        torch_dtype = kwargs.pop("torch_dtype", None)
1445
        use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False)
1446
1447
1448
1449
1450
1451

        # override default dtype if needed
        dtype_orig = None
        if torch_dtype is not None:
            dtype_orig = cls._set_default_torch_dtype(torch_dtype)

1452
        config = copy.deepcopy(config)  # We do not want to modify the config inplace in _from_config.
1453
1454
1455
1456
1457
1458
1459
1460
1461

        if config._attn_implementation_internal is not None:
            # In this case, the config has been created with the attn_implementation set by the user, which we
            # should respect.
            attn_implementation = config._attn_implementation_internal
        else:
            attn_implementation = None

        config._attn_implementation = kwargs.pop("attn_implementation", attn_implementation)
1462
        config = cls._autoset_attn_implementation(
1463
1464
1465
1466
            config,
            use_flash_attention_2=use_flash_attention_2,
            check_device_map=False,
            torch_dtype=torch_dtype,
1467
        )
1468

1469
1470
1471
1472
1473
1474
        if is_deepspeed_zero3_enabled():
            import deepspeed

            logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
            # this immediately partitions the model across all gpus, to avoid the overhead in time
            # and memory copying it on CPU or each GPU first
1475
            with deepspeed.zero.Init(config_dict_or_path=deepspeed_config()):
1476
                model = cls(config, **kwargs)
1477

1478
1479
1480
        else:
            model = cls(config, **kwargs)

1481
1482
1483
        # Flag for if we init with `zero3`, add an attr to the model so we can check downstream for issues
        model._transformers_zero3_init_used = is_deepspeed_zero3_enabled()

1484
1485
1486
1487
1488
1489
        # restore default dtype if it was modified
        if dtype_orig is not None:
            torch.set_default_dtype(dtype_orig)

        return model

1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
    @classmethod
    def _autoset_attn_implementation(
        cls,
        config,
        use_flash_attention_2: bool = False,
        torch_dtype: Optional[torch.dtype] = None,
        device_map: Optional[Union[str, Dict[str, int]]] = None,
        check_device_map: bool = True,
    ):
        """
        Automatically checks and dispatches to a default attention implementation. In order of priority:
            1. An implementation specified in `config._attn_implementation` (due for example to the argument attn_implementation="sdpa" in from_pretrained).
            2. DEPRECATED: if use_flash_attention_2 is set to `True` and `flash_attn` is available, flash attention. (`LlamaFlashAttention` for example)
            3. SDPA implementation, if available and supported by the model type. (`LlamaSdpaAttention` for example)
            4. The default model's implementation otherwise (`LlamaAttention` for example) .
        """
        # Here we use config._attn_implementation_internal to check whether the attention implementation was explicitely set by the user.
        # The property `PretrainedConfig._attn_implementation` is never `None`, for backward compatibility (always fall back on "eager").
        # The `hasattr` here is used as some Transformers tests for some reason do not call PretrainedConfig __init__ (e.g. test_no_super_init_config_and_model)
1509
        requested_attn_implementation = None
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
        if hasattr(config, "_attn_implementation_internal") and config._attn_implementation_internal is not None:
            if config._attn_implementation != "flash_attention_2" and use_flash_attention_2:
                raise ValueError(
                    f'Both attn_implementation="{config._attn_implementation}" and `use_flash_attention_2=True` were used when loading the model, which are not compatible.'
                    ' We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.'
                )

            if config._attn_implementation not in ["eager", "sdpa", "flash_attention_2"]:
                message = f'Specified `attn_implementation="{config._attn_implementation}"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)'
                if cls._supports_flash_attn_2:
                    message += ', `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)'
                if cls._supports_sdpa:
                    message += ', `"attn_implementation=sdpa"` (implementation using torch.nn.functional.scaled_dot_product_attention)'
                raise ValueError(message + ".")

            # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available.
1526
            requested_attn_implementation = config._attn_implementation_internal
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538

        if use_flash_attention_2:
            logger.warning_once(
                'The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.'
            )
            config._attn_implementation = "flash_attention_2"

        if config._attn_implementation == "flash_attention_2":
            cls._check_and_enable_flash_attn_2(
                config,
                torch_dtype=torch_dtype,
                device_map=device_map,
1539
                hard_check_only=False,
1540
1541
                check_device_map=check_device_map,
            )
1542
        elif requested_attn_implementation in [None, "sdpa"] and not is_torch_xla_available():
1543
            # use_flash_attention_2 takes priority over SDPA, hence SDPA treated in this elif.
1544
            config = cls._check_and_enable_sdpa(
1545
1546
                config,
                hard_check_only=False if requested_attn_implementation is None else True,
1547
            )
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557

            if (
                torch.version.hip is not None
                and config._attn_implementation == "sdpa"
                and torch.cuda.device_count() > 1
            ):
                logger.warning_once(
                    "Using the `SDPA` attention implementation on multi-gpu setup with ROCM may lead to performance issues due to the FA backend. Disabling it to use alternative backends."
                )
                torch.backends.cuda.enable_flash_sdp(False)
1558
        else:
1559
1560
1561
1562
            config._attn_implementation = "eager"

        return config

1563
1564
1565
1566
1567
1568
1569
    @classmethod
    def _set_default_torch_dtype(cls, dtype: torch.dtype) -> torch.dtype:
        """
        Change the default dtype and return the previous one. This is needed when wanting to instantiate the model
        under specific dtype.

        Args:
1570
            dtype (`torch.dtype`):
1571
1572
1573
                a floating dtype to set to.

        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
1574
1575
            `torch.dtype`: the original `dtype` that can be used to restore `torch.set_default_dtype(dtype)` if it was
            modified. If it wasn't, returns `None`.
1576

1577
1578
        Note `set_default_dtype` currently only works with floating-point types and asserts if for example,
        `torch.int64` is passed. So if a non-float `dtype` is passed this functions will throw an exception.
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
        """
        if not dtype.is_floating_point:
            raise ValueError(
                f"Can't instantiate {cls.__name__} model under dtype={dtype} since it is not a floating point dtype"
            )

        logger.info(f"Instantiating {cls.__name__} model under default dtype {dtype}.")
        dtype_orig = torch.get_default_dtype()
        torch.set_default_dtype(dtype)
        return dtype_orig

1590
    @property
1591
1592
    def base_model(self) -> nn.Module:
        """
1593
        `torch.nn.Module`: The main body of the model.
1594
        """
1595
        return getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
1596

1597
1598
    @classmethod
    def can_generate(cls) -> bool:
1599
1600
1601
1602
1603
1604
        """
        Returns whether this model can generate sequences with `.generate()`.

        Returns:
            `bool`: Whether this model can generate sequences with `.generate()`.
        """
1605
1606
1607
        # Detects whether `prepare_inputs_for_generation` has been overwritten, which is a requirement for generation.
        # Alternativelly, the model can also have a custom `generate` function.
        if "GenerationMixin" in str(cls.prepare_inputs_for_generation) and "GenerationMixin" in str(cls.generate):
1608
1609
1610
            return False
        return True

1611
1612
    @classmethod
    def _check_and_enable_flash_attn_2(
1613
1614
1615
1616
1617
1618
        cls,
        config,
        torch_dtype: Optional[torch.dtype] = None,
        device_map: Optional[Union[str, Dict[str, int]]] = None,
        check_device_map: bool = True,
        hard_check_only: bool = False,
1619
1620
    ) -> PretrainedConfig:
        """
1621
        Checks the availability of Flash Attention 2 and compatibility with the current model.
1622

1623
        If all checks pass and `hard_check_only` is False, the method will set the config attribute `attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module.
1624
1625
1626
        """
        if not cls._supports_flash_attn_2:
            raise ValueError(
1627
1628
1629
                f"{cls.__name__} does not support Flash Attention 2.0 yet. Please request to add support where"
                f" the model is hosted, on its model hub page: https://huggingface.co/{config._name_or_path}/discussions/new"
                " or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/new"
1630
1631
            )

1632
        if not is_flash_attn_2_available():
1633
1634
1635
            preface = "FlashAttention2 has been toggled on, but it cannot be used due to the following error:"
            install_message = "Please refer to the documentation of https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashattention-2 to install Flash Attention 2."

1636
1637
1638
1639
1640
            if importlib.util.find_spec("flash_attn") is None:
                raise ImportError(f"{preface} the package flash_attn seems to be not installed. {install_message}")

            flash_attention_version = version.parse(importlib.metadata.version("flash_attn"))
            if torch.version.cuda:
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
                if flash_attention_version < version.parse("2.1.0"):
                    raise ImportError(
                        f"{preface} you need flash_attn package version to be greater or equal than 2.1.0. Detected version {flash_attention_version}. {install_message}"
                    )
                else:
                    raise ImportError(f"{preface} Flash Attention 2 is not available. {install_message}")
            elif torch.version.hip:
                if flash_attention_version < version.parse("2.0.4"):
                    raise ImportError(
                        f"{preface} you need flash_attn package version to be greater or equal than 2.0.4. Make sure to have that version installed - detected version {flash_attention_version}. {install_message}"
                    )
                else:
                    raise ImportError(f"{preface} Flash Attention 2 is not available. {install_message}")
1654
1655
1656
1657
1658
1659
1660
1661
1662

        _is_bettertransformer = getattr(cls, "use_bettertransformer", False)

        if _is_bettertransformer:
            raise ValueError(
                "Flash Attention 2 and BetterTransformer API are not compatible. Please make sure to disable BetterTransformers by doing model.reverse_bettertransformer()"
            )

        if torch_dtype is None:
1663
            logger.warning_once(
1664
1665
1666
                "You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour"
            )
        elif torch_dtype is not None and torch_dtype not in [torch.float16, torch.bfloat16]:
1667
1668
1669
1670
            logger.warning_once(
                "Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but"
                f" the current dype in {cls.__name__} is {torch_dtype}. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator,"
                ' or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`'
1671
1672
            )

1673
1674
1675
        # The check `torch.empty(0).device.type != "cuda"` is needed as the model may be initialized after `torch.set_default_device` has been called,
        # or the model may be initialized under the context manager `with torch.device("cuda"):`.
        if check_device_map and device_map is None and torch.empty(0).device.type != "cuda":
1676
            if torch.cuda.is_available():
1677
                logger.warning_once(
1678
                    "You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU"
1679
1680
1681
1682
                    " after initializing it on CPU with `model.to('cuda')`."
                )
            else:
                raise ValueError(
1683
                    "You are attempting to use Flash Attention 2.0 with a model not initialized on GPU and with no GPU available. "
1684
1685
1686
1687
                    "This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map "
                    "or initialising the model on CPU and then moving it to GPU."
                )
        elif (
1688
1689
            check_device_map
            and device_map is not None
1690
1691
1692
1693
1694
1695
1696
            and isinstance(device_map, dict)
            and ("cpu" in device_map.values() or "disk" in device_map.values())
        ):
            raise ValueError(
                "You are attempting to use Flash Attention 2.0 with a model dispatched on CPU or disk. This is not supported. Please make sure to "
                "initialise the model on a GPU by passing a device_map that contains only GPU devices as keys."
            )
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
        if not hard_check_only:
            config._attn_implementation = "flash_attention_2"
        return config

    @classmethod
    def _check_and_enable_sdpa(cls, config, hard_check_only: bool = False) -> PretrainedConfig:
        """
        Checks the availability of SDPA for a given model.

        If all checks pass and `hard_check_only` is False, the method will set the config attribute `_attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module.
        """
        if hard_check_only:
            if not cls._supports_sdpa:
                raise ValueError(
1711
1712
1713
                    f"{cls.__name__} does not support an attention implementation through torch.nn.functional.scaled_dot_product_attention yet."
                    " Please request the support for this architecture: https://github.com/huggingface/transformers/issues/28005. If you believe"
                    ' this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`'
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
                )
            if not is_torch_sdpa_available():
                raise ImportError(
                    "PyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1."
                )

        if not is_torch_sdpa_available() or not cls._supports_sdpa:
            return config

        _is_bettertransformer = getattr(cls, "use_bettertransformer", False)
        if _is_bettertransformer:
            return config

        if not hard_check_only:
            config._attn_implementation = "sdpa"
1729
1730
        return config

1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
    def enable_input_require_grads(self):
        """
        Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping
        the model weights fixed.
        """

        def make_inputs_require_grads(module, input, output):
            output.requires_grad_(True)

        self._require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)

    def disable_input_require_grads(self):
        """
        Removes the `_require_grads_hook`.
        """
        self._require_grads_hook.remove()

1748
    def get_input_embeddings(self) -> nn.Module:
1749
1750
1751
1752
        """
        Returns the model's input embeddings.

        Returns:
1753
            `nn.Module`: A torch module mapping vocabulary to hidden states.
thomwolf's avatar
thomwolf committed
1754
        """
1755
        base_model = getattr(self, self.base_model_prefix, self)
thomwolf's avatar
thomwolf committed
1756
1757
1758
1759
        if base_model is not self:
            return base_model.get_input_embeddings()
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
1760

1761
    def set_input_embeddings(self, value: nn.Module):
1762
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
1763
        Set model's input embeddings.
1764
1765

        Args:
1766
            value (`nn.Module`): A module mapping vocabulary to hidden states.
thomwolf's avatar
thomwolf committed
1767
1768
1769
1770
1771
1772
        """
        base_model = getattr(self, self.base_model_prefix, self)
        if base_model is not self:
            base_model.set_input_embeddings(value)
        else:
            raise NotImplementedError
thomwolf's avatar
thomwolf committed
1773

1774
    def get_output_embeddings(self) -> nn.Module:
1775
1776
1777
1778
        """
        Returns the model's output embeddings.

        Returns:
1779
            `nn.Module`: A torch module mapping hidden states to vocabulary.
thomwolf's avatar
thomwolf committed
1780
        """
1781
        return None  # Overwrite for models with output embeddings
thomwolf's avatar
thomwolf committed
1782

1783
1784
    def _init_weights(self, module):
        """
1785
1786
1787
1788
        Initialize the weights. This method should be overridden by derived class and is
        the only initialization method that will be called when loading a checkpoint
        using `from_pretrained`. Any attempt to initialize outside of this function
        will be useless as the torch.nn.init function are all replaced with skip.
1789
        """
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
        pass

    def _initialize_weights(self, module):
        """
        Initialize the weights if they are not already initialized.
        """
        if getattr(module, "_is_hf_initialized", False):
            return
        self._init_weights(module)
        module._is_hf_initialized = True
1800

1801
    def tie_weights(self):
1802
1803
        """
        Tie the weights between the input embeddings and the output embeddings.
1804

Sylvain Gugger's avatar
Sylvain Gugger committed
1805
1806
        If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the
        weights instead.
thomwolf's avatar
thomwolf committed
1807
        """
1808
1809
1810
1811
        if getattr(self.config, "tie_word_embeddings", True):
            output_embeddings = self.get_output_embeddings()
            if output_embeddings is not None:
                self._tie_or_clone_weights(output_embeddings, self.get_input_embeddings())
thomwolf's avatar
thomwolf committed
1812

1813
        if getattr(self.config, "is_encoder_decoder", False) and getattr(self.config, "tie_encoder_decoder", False):
Weizhen's avatar
Weizhen committed
1814
1815
            if hasattr(self, self.base_model_prefix):
                self = getattr(self, self.base_model_prefix)
1816
1817
1818
1819
1820
1821
1822
            tied_weights = self._tie_encoder_decoder_weights(
                self.encoder, self.decoder, self.base_model_prefix, "encoder"
            )
            # Setting a dynamic variable instead of `_tied_weights_keys` because it's a class
            # attributed not an instance member, therefore modifying it will modify the entire class
            # Leading to issues on subsequent calls by different tests or subsequent calls.
            self._dynamic_tied_weights_keys = tied_weights
1823

Sylvain Gugger's avatar
Sylvain Gugger committed
1824
1825
1826
1827
        for module in self.modules():
            if hasattr(module, "_tie_weights"):
                module._tie_weights()

1828
    @staticmethod
1829
1830
1831
    def _tie_encoder_decoder_weights(
        encoder: nn.Module, decoder: nn.Module, base_model_prefix: str, base_encoder_name: str
    ):
1832
        uninitialized_encoder_weights: List[str] = []
1833
        tied_weights: List[str] = []
Weizhen's avatar
Weizhen committed
1834
1835
        if decoder.__class__ != encoder.__class__:
            logger.info(
Sylvain Gugger's avatar
Sylvain Gugger committed
1836
1837
                f"{decoder.__class__} and {encoder.__class__} are not equal. In this case make sure that all encoder"
                " weights are correctly initialized."
Weizhen's avatar
Weizhen committed
1838
            )
1839
1840
1841
1842
1843

        def tie_encoder_to_decoder_recursively(
            decoder_pointer: nn.Module,
            encoder_pointer: nn.Module,
            module_name: str,
1844
            base_encoder_name: str,
1845
1846
            uninitialized_encoder_weights: List[str],
            depth=0,
1847
1848
            total_decoder_name="",
            total_encoder_name="",
1849
1850
1851
        ):
            assert isinstance(decoder_pointer, nn.Module) and isinstance(
                encoder_pointer, nn.Module
1852
            ), f"{decoder_pointer} and {encoder_pointer} have to be of type nn.Module"
1853
1854
1855
            if hasattr(decoder_pointer, "weight"):
                assert hasattr(encoder_pointer, "weight")
                encoder_pointer.weight = decoder_pointer.weight
1856
                tied_weights.append(f"{base_encoder_name}{total_encoder_name}.weight")
1857
1858
                if hasattr(decoder_pointer, "bias"):
                    assert hasattr(encoder_pointer, "bias")
1859
                    tied_weights.append(f"{base_encoder_name}{total_encoder_name}.bias")
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
                    encoder_pointer.bias = decoder_pointer.bias
                return

            encoder_modules = encoder_pointer._modules
            decoder_modules = decoder_pointer._modules
            if len(decoder_modules) > 0:
                assert (
                    len(encoder_modules) > 0
                ), f"Encoder module {encoder_pointer} does not match decoder module {decoder_pointer}"

1870
                all_encoder_weights = {module_name + "/" + sub_name for sub_name in encoder_modules.keys()}
1871
1872
1873
1874
1875
                encoder_layer_pos = 0
                for name, module in decoder_modules.items():
                    if name.isdigit():
                        encoder_name = str(int(name) + encoder_layer_pos)
                        decoder_name = name
Weizhen's avatar
Weizhen committed
1876
1877
1878
                        if not isinstance(decoder_modules[decoder_name], type(encoder_modules[encoder_name])) and len(
                            encoder_modules
                        ) != len(decoder_modules):
1879
1880
                            # this can happen if the name corresponds to the position in a list module list of layers
                            # in this case the decoder has added a cross-attention that the encoder does not have
1881
                            # thus skip this step and subtract one layer pos from encoder
1882
1883
1884
1885
1886
1887
                            encoder_layer_pos -= 1
                            continue
                    elif name not in encoder_modules:
                        continue
                    elif depth > 500:
                        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
1888
1889
                            "Max depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is"
                            " a circular dependency between two or more `nn.Modules` of your model."
1890
1891
1892
1893
1894
1895
1896
                        )
                    else:
                        decoder_name = encoder_name = name
                    tie_encoder_to_decoder_recursively(
                        decoder_modules[decoder_name],
                        encoder_modules[encoder_name],
                        module_name + "/" + name,
1897
                        base_encoder_name,
1898
1899
                        uninitialized_encoder_weights,
                        depth=depth + 1,
1900
1901
                        total_encoder_name=f"{total_encoder_name}.{encoder_name}",
                        total_decoder_name=f"{total_decoder_name}.{decoder_name}",
1902
1903
1904
1905
1906
1907
                    )
                    all_encoder_weights.remove(module_name + "/" + encoder_name)

                uninitialized_encoder_weights += list(all_encoder_weights)

        # tie weights recursively
1908
1909
1910
1911
        tie_encoder_to_decoder_recursively(
            decoder, encoder, base_model_prefix, base_encoder_name, uninitialized_encoder_weights
        )

1912
1913
1914
1915
        if len(uninitialized_encoder_weights) > 0:
            logger.warning(
                f"The following encoder weights were not tied to the decoder {uninitialized_encoder_weights}"
            )
1916
        return tied_weights
1917

1918
    def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
Lysandre's avatar
Lysandre committed
1919
        """Tie or clone module weights depending of whether we are using TorchScript or not"""
thomwolf's avatar
thomwolf committed
1920
        if self.config.torchscript:
1921
            output_embeddings.weight = nn.Parameter(input_embeddings.weight.clone())
thomwolf's avatar
thomwolf committed
1922
        else:
1923
            output_embeddings.weight = input_embeddings.weight
thomwolf's avatar
thomwolf committed
1924

Sam Shleifer's avatar
Sam Shleifer committed
1925
        if getattr(output_embeddings, "bias", None) is not None:
1926
            output_embeddings.bias.data = nn.functional.pad(
1927
                output_embeddings.bias.data,
Lysandre's avatar
Lysandre committed
1928
1929
1930
1931
                (
                    0,
                    output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0],
                ),
1932
1933
                "constant",
                0,
1934
            )
1935
        if hasattr(output_embeddings, "out_features") and hasattr(input_embeddings, "num_embeddings"):
1936
            output_embeddings.out_features = input_embeddings.num_embeddings
1937

Marc Sun's avatar
Marc Sun committed
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
    def _get_no_split_modules(self, device_map: str):
        """
        Get the modules of the model that should not be spit when using device_map. We iterate through the modules to
        get the underlying `_no_split_modules`.

        Args:
            device_map (`str`):
                The device map value. Options are ["auto", "balanced", "balanced_low_0", "sequential"]

        Returns:
            `List[str]`: List of modules that should not be split
        """
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
        _no_split_modules = set()
        modules_to_check = [self]
        while len(modules_to_check) > 0:
            module = modules_to_check.pop(-1)
            # if the module does not appear in _no_split_modules, we also check the children
            if module.__class__.__name__ not in _no_split_modules:
                if isinstance(module, PreTrainedModel):
                    if module._no_split_modules is None:
                        raise ValueError(
                            f"{module.__class__.__name__} does not support `device_map='{device_map}'`. To implement support, the model "
                            "class needs to implement the `_no_split_modules` attribute."
                        )
                    else:
                        _no_split_modules = _no_split_modules | set(module._no_split_modules)
                modules_to_check += list(module.children())
Marc Sun's avatar
Marc Sun committed
1965
1966
        return list(_no_split_modules)

1967
1968
1969
    def resize_token_embeddings(
        self, new_num_tokens: Optional[int] = None, pad_to_multiple_of: Optional[int] = None
    ) -> nn.Embedding:
1970
        """
1971
        Resizes input token embeddings matrix of the model if `new_num_tokens != config.vocab_size`.
1972

1973
        Takes care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.
thomwolf's avatar
thomwolf committed
1974

1975
        Arguments:
1976
            new_num_tokens (`int`, *optional*):
1977
                The new number of tokens in the embedding matrix. Increasing the size will add newly initialized
Sylvain Gugger's avatar
Sylvain Gugger committed
1978
1979
                vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just
                returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything.
1980
            pad_to_multiple_of (`int`, *optional*):
Arthur's avatar
Arthur committed
1981
1982
                If set will pad the embedding matrix to a multiple of the provided value.If `new_num_tokens` is set to
                `None` will just pad the embedding to a multiple of `pad_to_multiple_of`.
1983
1984
1985
1986
1987

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
                details about this, or help on choosing the correct value for resizing, refer to this guide:
                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc
1988
1989

        Return:
1990
            `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model.
thomwolf's avatar
thomwolf committed
1991
        """
1992
        model_embeds = self._resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
Arthur's avatar
Arthur committed
1993
        if new_num_tokens is None and pad_to_multiple_of is None:
thomwolf's avatar
thomwolf committed
1994
            return model_embeds
thomwolf's avatar
thomwolf committed
1995

1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
        # Since we are basically resuing the same old embeddings with new weight values, gathering is required
        is_quantized = hasattr(self, "hf_quantizer") and self.hf_quantizer is not None
        if is_deepspeed_zero3_enabled() and not is_quantized:
            import deepspeed

            with deepspeed.zero.GatheredParameters(model_embeds.weight, modifier_rank=None):
                vocab_size = model_embeds.weight.shape[0]
        else:
            vocab_size = model_embeds.weight.shape[0]

thomwolf's avatar
thomwolf committed
2006
        # Update base model and current model config
2007
        if hasattr(self.config, "text_config"):
2008
            self.config.text_config.vocab_size = vocab_size
2009
        else:
2010
2011
            self.config.vocab_size = vocab_size
        self.vocab_size = vocab_size
thomwolf's avatar
thomwolf committed
2012
2013

        # Tie weights again if needed
2014
        self.tie_weights()
thomwolf's avatar
thomwolf committed
2015

thomwolf's avatar
thomwolf committed
2016
2017
        return model_embeds

2018
    def _resize_token_embeddings(self, new_num_tokens, pad_to_multiple_of=None):
thomwolf's avatar
thomwolf committed
2019
        old_embeddings = self.get_input_embeddings()
2020
        new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens, pad_to_multiple_of)
2021
2022
2023
        if hasattr(old_embeddings, "_hf_hook"):
            hook = old_embeddings._hf_hook
            add_hook_to_module(new_embeddings, hook)
2024
2025
        old_embeddings_requires_grad = old_embeddings.weight.requires_grad
        new_embeddings.requires_grad_(old_embeddings_requires_grad)
thomwolf's avatar
thomwolf committed
2026
        self.set_input_embeddings(new_embeddings)
2027
        is_quantized = hasattr(self, "hf_quantizer") and self.hf_quantizer is not None
2028

2029
2030
        # Update new_num_tokens with the actual size of new_embeddings
        if pad_to_multiple_of is not None:
2031
            if is_deepspeed_zero3_enabled() and not is_quantized:
2032
2033
2034
2035
2036
2037
2038
                import deepspeed

                with deepspeed.zero.GatheredParameters(new_embeddings.weight, modifier_rank=None):
                    new_num_tokens = new_embeddings.weight.shape[0]
            else:
                new_num_tokens = new_embeddings.weight.shape[0]

2039
2040
2041
        # if word embeddings are not tied, make sure that lm head is resized as well
        if self.get_output_embeddings() is not None and not self.config.tie_word_embeddings:
            old_lm_head = self.get_output_embeddings()
2042
2043
2044
2045
            if isinstance(old_lm_head, torch.nn.Embedding):
                new_lm_head = self._get_resized_embeddings(old_lm_head, new_num_tokens)
            else:
                new_lm_head = self._get_resized_lm_head(old_lm_head, new_num_tokens)
2046
2047
2048
            if hasattr(old_lm_head, "_hf_hook"):
                hook = old_lm_head._hf_hook
                add_hook_to_module(new_lm_head, hook)
2049
2050
            old_lm_head_requires_grad = old_lm_head.weight.requires_grad
            new_lm_head.requires_grad_(old_lm_head_requires_grad)
2051
2052
            self.set_output_embeddings(new_lm_head)

thomwolf's avatar
thomwolf committed
2053
        return self.get_input_embeddings()
2054

2055
    def _get_resized_embeddings(
2056
2057
2058
2059
        self,
        old_embeddings: nn.Embedding,
        new_num_tokens: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
2060
    ) -> nn.Embedding:
2061
2062
2063
        """
        Build a resized Embedding Module from a provided token Embedding Module. Increasing the size will add newly
        initialized vectors at the end. Reducing the size will remove vectors from the end
2064
2065

        Args:
2066
            old_embeddings (`torch.nn.Embedding`):
2067
                Old embeddings to be resized.
2068
            new_num_tokens (`int`, *optional*):
2069
                New number of tokens in the embedding matrix.
2070
2071

                Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
2072
                vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
2073
                `torch.nn.Embedding` module of the model without doing anything.
2074
            pad_to_multiple_of (`int`, *optional*):
Arthur's avatar
Arthur committed
2075
2076
                If set will pad the embedding matrix to a multiple of the provided value. If `new_num_tokens` is set to
                `None` will just pad the embedding to a multiple of `pad_to_multiple_of`.
2077
2078
2079
2080
2081
2082

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more
                details about this, or help on choosing the correct value for resizing, refer to this guide:
                https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc

2083
2084

        Return:
2085
2086
            `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if
            `new_num_tokens` is `None`
2087
        """
2088
2089
2090
2091
2092
2093
2094
2095

        if pad_to_multiple_of is not None:
            if not isinstance(pad_to_multiple_of, int):
                raise ValueError(
                    f"Asking to pad the embedding matrix to a multiple of `{pad_to_multiple_of}`, which is not and integer. Please make sure to pass an integer"
                )
            if new_num_tokens is None:
                new_num_tokens = old_embeddings.weight.shape[0]
2096
            new_num_tokens = ((new_num_tokens + pad_to_multiple_of - 1) // pad_to_multiple_of) * pad_to_multiple_of
2097
        else:
2098
            logger.info(
2099
                "You are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding"
2100
                f" dimension will be {new_num_tokens}. This might induce some performance reduction as *Tensor Cores* will not be available."
2101
                " For more details about this, or help on choosing the correct value for resizing, refer to this guide:"
2102
2103
2104
                " https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc"
            )

2105
2106
2107
        if new_num_tokens is None:
            return old_embeddings

2108
2109
        is_quantized = hasattr(self, "hf_quantizer") and self.hf_quantizer is not None
        if is_deepspeed_zero3_enabled() and not is_quantized:
2110
2111
2112
2113
2114
2115
2116
            import deepspeed

            with deepspeed.zero.GatheredParameters(old_embeddings.weight, modifier_rank=None):
                old_num_tokens, old_embedding_dim = old_embeddings.weight.size()
        else:
            old_num_tokens, old_embedding_dim = old_embeddings.weight.size()

2117
        if old_num_tokens == new_num_tokens and not is_deepspeed_zero3_enabled():
2118
2119
            return old_embeddings

2120
2121
        if not isinstance(old_embeddings, nn.Embedding):
            raise TypeError(
Sylvain Gugger's avatar
Sylvain Gugger committed
2122
2123
2124
                f"Old embeddings are of type {type(old_embeddings)}, which is not an instance of {nn.Embedding}. You"
                " should either use a different resize function or make sure that `old_embeddings` are an instance of"
                f" {nn.Embedding}."
2125
2126
            )

2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
        # Build new embeddings

        # When using DeepSpeed ZeRO-3, we shouldn't create new embeddings with DeepSpeed init
        # because the shape of the new embedding layer is used across various modeling files
        # as well as to update config vocab size. Shape will be 0 when using DeepSpeed init leading
        # to errors when training.
        new_embeddings = nn.Embedding(
            new_num_tokens,
            old_embedding_dim,
            device=old_embeddings.weight.device,
            dtype=old_embeddings.weight.dtype,
        )

        # initialize all new embeddings (in particular added tokens)
        self._init_weights(new_embeddings)

        # Copy token embeddings from the previous weights

2145
2146
        # numbers of tokens to copy
        n = min(old_num_tokens, new_num_tokens)
2147

2148
        if is_deepspeed_zero3_enabled() and not is_quantized:
2149
2150
            import deepspeed

2151
2152
2153
            params = [old_embeddings.weight, new_embeddings.weight]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
                new_embeddings.weight.data[:n, :] = old_embeddings.weight.data[:n, :]
2154
2155
        else:
            new_embeddings.weight.data[:n, :] = old_embeddings.weight.data[:n, :]
2156

2157
2158
2159
        # Replace weights in old_embeddings and return to maintain the same embedding type.
        # This ensures correct functionality when a Custom Embedding class is passed as input.
        # The input and output embedding types remain consistent. (c.f. https://github.com/huggingface/transformers/pull/31979)
2160
2161
2162
2163
2164
        if is_deepspeed_zero3_enabled() and not is_quantized:
            import deepspeed

            params = [old_embeddings.weight, new_embeddings.weight]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
2165
                old_embeddings.weight = new_embeddings.weight
2166
                old_embeddings.num_embeddings = new_embeddings.weight.data.shape[0]
2167

2168
2169
2170
2171
2172
2173
2174
2175
2176
                # If the new number of tokens is smaller than the original `padding_idx`, the `padding_idx`
                # will be set to `None` in the resized embeddings.
                if old_embeddings.padding_idx is not None and (new_num_tokens - 1) < old_embeddings.padding_idx:
                    old_embeddings.padding_idx = None
        else:
            old_embeddings.weight.data = new_embeddings.weight.data
            old_embeddings.num_embeddings = new_embeddings.weight.data.shape[0]
            if old_embeddings.padding_idx is not None and (new_num_tokens - 1) < old_embeddings.padding_idx:
                old_embeddings.padding_idx = None
2177
2178

        return old_embeddings
2179

2180
    def _get_resized_lm_head(
2181
2182
        self, old_lm_head: nn.Linear, new_num_tokens: Optional[int] = None, transposed: Optional[bool] = False
    ) -> nn.Linear:
2183
2184
2185
2186
2187
        """
        Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized
        vectors at the end. Reducing the size will remove vectors from the end

        Args:
2188
            old_lm_head (`torch.nn.Linear`):
2189
                Old lm head liner layer to be resized.
2190
            new_num_tokens (`int`, *optional*):
2191
2192
2193
                New number of tokens in the linear matrix.

                Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
2194
                vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
2195
2196
2197
                `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults
                to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim,
                vocab_size` else `vocab_size, lm_head_dim`.
2198
2199

        Return:
Sylvain Gugger's avatar
Sylvain Gugger committed
2200
2201
            `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is
            `None`
2202
2203
2204
2205
        """
        if new_num_tokens is None:
            return old_lm_head

2206
2207
        is_quantized = hasattr(self, "hf_quantizer") and self.hf_quantizer is not None
        if is_deepspeed_zero3_enabled() and not is_quantized:
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
            import deepspeed

            with deepspeed.zero.GatheredParameters(old_lm_head.weight, modifier_rank=None):
                old_num_tokens, old_lm_head_dim = (
                    old_lm_head.weight.size() if not transposed else old_lm_head.weight.t().size()
                )
        else:
            old_num_tokens, old_lm_head_dim = (
                old_lm_head.weight.size() if not transposed else old_lm_head.weight.t().size()
            )
2218

2219
        if old_num_tokens == new_num_tokens and not is_deepspeed_zero3_enabled():
2220
2221
2222
2223
            return old_lm_head

        if not isinstance(old_lm_head, nn.Linear):
            raise TypeError(
Sylvain Gugger's avatar
Sylvain Gugger committed
2224
2225
2226
                f"Old language model head is of type {type(old_lm_head)}, which is not an instance of {nn.Linear}. You"
                " should either use a different resize function or make sure that `old_lm_head` are an instance of"
                f" {nn.Linear}."
2227
2228
2229
2230
2231
2232
            )

        # Build new lm head
        new_lm_head_shape = (old_lm_head_dim, new_num_tokens) if not transposed else (new_num_tokens, old_lm_head_dim)
        has_new_lm_head_bias = old_lm_head.bias is not None

2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
        # When using DeepSpeed ZeRO-3, we shouldn't create new embeddings with DeepSpeed init
        # because the shape of the new embedding layer is used across various modeling files
        # as well as to update config vocab size. Shape will be 0 when using DeepSpeed init leading
        # to errors when training.
        new_lm_head = nn.Linear(
            *new_lm_head_shape,
            bias=has_new_lm_head_bias,
            device=old_lm_head.weight.device,
            dtype=old_lm_head.weight.dtype,
        )

        # initialize new lm head (in particular added tokens)
        self._init_weights(new_lm_head)

2247
2248
        num_tokens_to_copy = min(old_num_tokens, new_num_tokens)

2249
        if is_deepspeed_zero3_enabled() and not is_quantized:
2250
2251
            import deepspeed

2252
2253
            params = [old_lm_head.weight, old_lm_head.bias, new_lm_head.weight, new_lm_head.bias]
            with deepspeed.zero.GatheredParameters(params, modifier_rank=0):
2254
2255
2256
                self._copy_lm_head_original_to_resized(
                    new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
                )
2257
        else:
2258
2259
            self._copy_lm_head_original_to_resized(
                new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
2260
            )
2261
2262
2263

        return new_lm_head

2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
    def _copy_lm_head_original_to_resized(
        self, new_lm_head, old_lm_head, num_tokens_to_copy, transposed, has_new_lm_head_bias
    ):
        # Copy old lm head weights to new lm head
        if not transposed:
            new_lm_head.weight.data[:num_tokens_to_copy, :] = old_lm_head.weight.data[:num_tokens_to_copy, :]
        else:
            new_lm_head.weight.data[:, :num_tokens_to_copy] = old_lm_head.weight.data[:, :num_tokens_to_copy]

        # Copy bias weights to new lm head
        if has_new_lm_head_bias:
            new_lm_head.bias.data[:num_tokens_to_copy] = old_lm_head.bias.data[:num_tokens_to_copy]

2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
    def resize_position_embeddings(self, new_num_position_embeddings: int):
        raise NotImplementedError(
            f"`resize_position_embeddings` is not implemented for {self.__class__}`. To implement it, you should "
            f"overwrite this method in the class {self.__class__} in `modeling_{self.__class__.__module__}.py`"
        )

    def get_position_embeddings(self) -> Union[nn.Embedding, Tuple[nn.Embedding]]:
        raise NotImplementedError(
            f"`get_position_embeddings` is not implemented for {self.__class__}`. To implement it, you should "
            f"overwrite this method in the class {self.__class__} in `modeling_{self.__class__.__module__}.py`"
        )

2289
    def init_weights(self):
2290
        """
2291
2292
        If needed prunes and maybe initializes weights. If using a custom `PreTrainedModel`, you need to implement any
        initialization logic in `_init_weights`.
2293
        """
2294
2295
2296
2297
        # Prune heads if needed
        if self.config.pruned_heads:
            self.prune_heads(self.config.pruned_heads)

2298
2299
        if _init_weights:
            # Initialize weights
2300
            self.apply(self._initialize_weights)
2301
2302
2303
2304

            # Tie weights should be skipped when not initializing all weights
            # since from_pretrained(...) calls tie weights anyways
            self.tie_weights()
2305

2306
2307
2308
    def prune_heads(self, heads_to_prune: Dict[int, List[int]]):
        """
        Prunes heads of the base model.
2309

2310
        Arguments:
2311
            heads_to_prune (`Dict[int, List[int]]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2312
2313
2314
                Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads
                to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on
                layer 1 and heads 2 and 3 on layer 2.
thomwolf's avatar
thomwolf committed
2315
        """
2316
        # save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads
2317
        for layer, heads in heads_to_prune.items():
2318
2319
2320
            union_heads = set(self.config.pruned_heads.get(layer, [])) | set(heads)
            self.config.pruned_heads[layer] = list(union_heads)  # Unfortunately we have to store it as list for JSON

2321
        self.base_model._prune_heads(heads_to_prune)
thomwolf's avatar
thomwolf committed
2322

2323
    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
2324
2325
2326
2327
2328
        """
        Activates gradient checkpointing for the current model.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
2329
2330
2331
2332
2333
2334
2335

        We pass the `__call__` method of the modules instead of `forward` because `__call__` attaches all the hooks of
        the module. https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2

        Args:
            gradient_checkpointing_kwargs (dict, *optional*):
                Additional keyword arguments passed along to the `torch.utils.checkpoint.checkpoint` function.
2336
2337
2338
        """
        if not self.supports_gradient_checkpointing:
            raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
2339
2340

        if gradient_checkpointing_kwargs is None:
2341
            gradient_checkpointing_kwargs = {"use_reentrant": True}
2342

2343
        gradient_checkpointing_func = functools.partial(checkpoint, **gradient_checkpointing_kwargs)
2344

2345
        # For old GC format (transformers < 4.35.0) for models that live on the Hub
Stas Bekman's avatar
Stas Bekman committed
2346
        # we will fall back to the overwritten `_set_gradient_checkpointing` method
2347
2348
2349
2350
2351
2352
        _is_using_old_format = "value" in inspect.signature(self._set_gradient_checkpointing).parameters

        if not _is_using_old_format:
            self._set_gradient_checkpointing(enable=True, gradient_checkpointing_func=gradient_checkpointing_func)
        else:
            self.apply(partial(self._set_gradient_checkpointing, value=True))
2353
            logger.warning(
2354
2355
2356
                "You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it)."
                "Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model."
            )
2357

2358
2359
2360
2361
2362
2363
2364
        if getattr(self, "_hf_peft_config_loaded", False):
            # When using PEFT + gradient checkpointing + Trainer we need to make sure the input has requires_grad=True
            # we do it also on PEFT: https://github.com/huggingface/peft/blob/85013987aa82aa1af3da1236b6902556ce3e483e/src/peft/peft_model.py#L334
            # When training with PEFT, only LoRA layers will have requires grad set to True, but the output of frozen layers need to propagate
            # the gradients to make sure the gradient flows.
            self.enable_input_require_grads()

2365
    def _set_gradient_checkpointing(self, enable: bool = True, gradient_checkpointing_func: Callable = checkpoint):
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
        is_gradient_checkpointing_set = False

        # Apply it on the top-level module in case the top-level modules supports it
        # for example, LongT5Stack inherits from `PreTrainedModel`.
        if hasattr(self, "gradient_checkpointing"):
            self._gradient_checkpointing_func = gradient_checkpointing_func
            self.gradient_checkpointing = enable
            is_gradient_checkpointing_set = True

        for module in self.modules():
            if hasattr(module, "gradient_checkpointing"):
                module._gradient_checkpointing_func = gradient_checkpointing_func
                module.gradient_checkpointing = enable
                is_gradient_checkpointing_set = True

        if not is_gradient_checkpointing_set:
            raise ValueError(
                f"{self.__class__.__name__} is not compatible with gradient checkpointing. Make sure all the architecture support it by setting a boolean attribute"
                " `gradient_checkpointing` to modules of the model that uses checkpointing."
            )

2387
    def gradient_checkpointing_disable(self):
2388
2389
2390
2391
2392
2393
2394
        """
        Deactivates gradient checkpointing for the current model.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        if self.supports_gradient_checkpointing:
2395
2396
2397
2398
2399
2400
            # For old GC format (transformers < 4.35.0) for models that live on the Hub
            # we will fall back to the overwritten `_set_gradient_checkpointing` methid
            _is_using_old_format = "value" in inspect.signature(self._set_gradient_checkpointing).parameters
            if not _is_using_old_format:
                self._set_gradient_checkpointing(enable=False)
            else:
2401
                logger.warning(
2402
2403
2404
2405
                    "You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it)."
                    "Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model."
                )
                self.apply(partial(self._set_gradient_checkpointing, value=False))
2406

2407
2408
2409
        if getattr(self, "_hf_peft_config_loaded", False):
            self.disable_input_require_grads()

2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
    @property
    def is_gradient_checkpointing(self) -> bool:
        """
        Whether gradient checkpointing is activated for this model or not.

        Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint
        activations".
        """
        return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())

2420
2421
2422
    def save_pretrained(
        self,
        save_directory: Union[str, os.PathLike],
2423
        is_main_process: bool = True,
2424
2425
        state_dict: Optional[dict] = None,
        save_function: Callable = torch.save,
Sylvain Gugger's avatar
Sylvain Gugger committed
2426
        push_to_hub: bool = False,
2427
        max_shard_size: Union[int, str] = "5GB",
2428
        safe_serialization: bool = True,
2429
        variant: Optional[str] = None,
2430
        token: Optional[Union[str, bool]] = None,
2431
        save_peft_format: bool = True,
Sylvain Gugger's avatar
Sylvain Gugger committed
2432
        **kwargs,
2433
    ):
2434
2435
        """
        Save a model and its configuration file to a directory, so that it can be re-loaded using the
2436
        [`~PreTrainedModel.from_pretrained`] class method.
2437

2438
        Arguments:
2439
            save_directory (`str` or `os.PathLike`):
2440
                Directory to which to save. Will be created if it doesn't exist.
2441
2442
2443
2444
            is_main_process (`bool`, *optional*, defaults to `True`):
                Whether the process calling this is the main process or not. Useful when in distributed training like
                TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on
                the main process to avoid race conditions.
2445
            state_dict (nested dictionary of `torch.Tensor`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2446
2447
2448
                The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only
                save parts of the model or if special precautions need to be taken when recovering the state dictionary
                of a model (like when using model parallelism).
2449
            save_function (`Callable`):
2450
                The function to use to save the state dictionary. Useful on distributed training like TPUs when one
2451
2452
                need to replace `torch.save` by another method.
            push_to_hub (`bool`, *optional*, defaults to `False`):
2453
2454
2455
                Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
                repository you want to push to with `repo_id` (will default to the name of `save_directory` in your
                namespace).
2456
            max_shard_size (`int` or `str`, *optional*, defaults to `"5GB"`):
Sylvain Gugger's avatar
Sylvain Gugger committed
2457
2458
                The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
                lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
2459
2460
                We default it to 5GB in order for models to be able to run easily on free-tier google colab instances
                without CPU OOM issues.
Sylvain Gugger's avatar
Sylvain Gugger committed
2461
2462
2463
2464
2465
2466
2467
2468

                <Tip warning={true}>

                If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
                which will be bigger than `max_shard_size`.

                </Tip>

2469
            safe_serialization (`bool`, *optional*, defaults to `True`):
2470
                Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
2471
2472
            variant (`str`, *optional*):
                If specified, weights are saved in the format pytorch_model.<variant>.bin.
2473
2474
2475
            token (`str` or `bool`, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
2476
2477
2478
2479
            save_peft_format (`bool`, *optional*, defaults to `True`):
                For backward compatibility with PEFT library, in case adapter weights are attached to the model, all
                keys of the state dict of adapters needs to be pre-pended with `base_model.model`. Advanced users can
                disable this behaviours by setting `save_peft_format` to `False`.
2480
            kwargs (`Dict[str, Any]`, *optional*):
2481
                Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
2482
        """
2483
        use_auth_token = kwargs.pop("use_auth_token", None)
2484
        ignore_metadata_errors = kwargs.pop("ignore_metadata_errors", False)
2485
2486
2487

        if use_auth_token is not None:
            warnings.warn(
2488
2489
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        if token is not None:
            kwargs["token"] = token

Younes Belkada's avatar
Younes Belkada committed
2500
2501
        _hf_peft_config_loaded = getattr(self, "_hf_peft_config_loaded", False)

2502
2503
2504
2505
        hf_quantizer = getattr(self, "hf_quantizer", None)
        quantization_serializable = (
            hf_quantizer is not None and isinstance(hf_quantizer, HfQuantizer) and hf_quantizer.is_serializable
        )
2506

2507
2508
2509
2510
        if hf_quantizer is not None and not _hf_peft_config_loaded and not quantization_serializable:
            raise ValueError(
                f"The model is quantized with {hf_quantizer.quantization_config.quant_method} and is not serializable - check out the warnings from"
                " the logger on the traceback to understand the reason why the quantized model is not serializable."
2511
2512
            )

2513
2514
2515
2516
2517
        if "save_config" in kwargs:
            warnings.warn(
                "`save_config` is deprecated and will be removed in v5 of Transformers. Use `is_main_process` instead."
            )
            is_main_process = kwargs.pop("save_config")
2518
2519
        if safe_serialization and not is_safetensors_available():
            raise ImportError("`safe_serialization` requires the `safetensors library: `pip install safetensors`.")
2520

2521
        if os.path.isfile(save_directory):
2522
            logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
2523
            return
2524

2525
2526
        os.makedirs(save_directory, exist_ok=True)

2527
2528
        if push_to_hub:
            commit_message = kwargs.pop("commit_message", None)
2529
            repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
2530
            repo_id = self._create_repo(repo_id, **kwargs)
2531
            files_timestamps = self._get_files_timestamps(save_directory)
2532

Julien Chaumond's avatar
Julien Chaumond committed
2533
        # Only save the model itself if we are using distributed training
2534
        model_to_save = unwrap_model(self)
2535

2536
2537
2538
2539
2540
        # save the string version of dtype to the config, e.g. convert torch.float32 => "float32"
        # we currently don't use this setting automatically, but may start to use with v5
        dtype = get_parameter_dtype(model_to_save)
        model_to_save.config.torch_dtype = str(dtype).split(".")[1]

Julien Chaumond's avatar
Julien Chaumond committed
2541
2542
2543
        # Attach architecture to the config
        model_to_save.config.architectures = [model_to_save.__class__.__name__]

2544
2545
2546
2547
2548
        # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be
        # loaded from the Hub.
        if self._auto_class is not None:
            custom_object_save(self, save_directory, config=self.config)

2549
        # Save the config
2550
        if is_main_process:
2551
2552
            if not _hf_peft_config_loaded:
                model_to_save.config.save_pretrained(save_directory)
2553
            if self.can_generate():
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
                # generation config built from the model config + the model config holds generation kwargs -> generate
                # may revert to legacy behavior if the two don't match
                if (
                    model_to_save.generation_config._from_model_config
                    and model_to_save.config._has_non_default_generation_parameters()
                ):
                    new_generation_config = GenerationConfig.from_model_config(model_to_save.config)
                    if new_generation_config != model_to_save.generation_config:
                        logger.warning(
                            "Your generation config was originally created from the model config, but the model "
                            "config has changed since then. Unless you pass the `generation_config` argument to this "
                            "model's `generate` calls, they will revert to the legacy behavior where the base "
                            "`generate` parameterization is loaded from the model config instead. "
                            "To avoid this behavior and this warning, we recommend you to overwrite the generation "
                            "config model attribute before calling the model's `save_pretrained`, preferably also "
                            "removing any generation kwargs from the model config. This warning will be raised to an "
                            "exception in v4.41."
                        )
2572
                model_to_save.generation_config.save_pretrained(save_directory)
2573

2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
            if _hf_peft_config_loaded:
                logger.info(
                    "Detected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved."
                )
                state_dict = model_to_save.get_adapter_state_dict()

                if save_peft_format:
                    logger.info(
                        "To match the expected format of the PEFT library, all keys of the state dict of adapters will be pre-pended with `base_model.model`."
                    )
                    peft_state_dict = {}
                    for key, value in state_dict.items():
                        peft_state_dict[f"base_model.model.{key}"] = value
                    state_dict = peft_state_dict

2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
                active_adapter = self.active_adapters()

                if len(active_adapter) > 1:
                    raise ValueError(
                        "Multiple active adapters detected, saving multiple active adapters is not supported yet. You can save adapters separately one by one "
                        "by iteratively calling `model.set_adapter(adapter_name)` then `model.save_pretrained(...)`"
                    )
                active_adapter = active_adapter[0]

                current_peft_config = self.peft_config[active_adapter]
2599
2600
                current_peft_config.save_pretrained(save_directory)

2601
2602
2603
        # for offloaded modules
        module_map = {}

2604
2605
        # Save the model
        if state_dict is None:
2606
2607
2608
2609
2610
            # if any model parameters are offloaded, make module map
            if (
                hasattr(self, "hf_device_map")
                and len(set(self.hf_device_map.values())) > 1
                and ("cpu" in self.hf_device_map.values() or "disk" in self.hf_device_map.values())
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
            ):
                warnings.warn(
                    "Attempting to save a model with offloaded modules. Ensure that unallocated cpu memory exceeds the `shard_size` (5GB default)"
                )
                for name, module in model_to_save.named_modules():
                    if name == "":
                        continue
                    module_state_dict = module.state_dict()

                    for key in module_state_dict:
                        module_map[name + f".{key}"] = module
2622
            state_dict = model_to_save.state_dict()
2623

2624
2625
2626
2627
2628
        # Translate state_dict from smp to hf if saving with smp >= 1.10
        if IS_SAGEMAKER_MP_POST_1_10:
            for smp_to_hf, _ in smp.state.module_manager.translate_functions:
                state_dict = smp_to_hf(state_dict)

2629
        # Handle the case where some state_dict keys shouldn't be saved
2630
        if self._keys_to_ignore_on_save is not None:
2631
            for ignore_key in self._keys_to_ignore_on_save:
2632
2633
                if ignore_key in state_dict.keys():
                    del state_dict[ignore_key]
2634
2635
2636
2637
2638
        if safe_serialization:
            # Safetensors does not allow tensor aliasing.
            # We're going to remove aliases before saving
            ptrs = collections.defaultdict(list)
            for name, tensor in state_dict.items():
2639
2640
2641
2642
2643
2644
2645
                # Sometimes in the state_dict we have non-tensor objects.
                # e.g. in bitsandbytes we have some `str` objects in the state_dict
                if isinstance(tensor, torch.Tensor):
                    ptrs[id_tensor_storage(tensor)].append(name)
                else:
                    # In the non-tensor case, fall back to the pointer of the object itself
                    ptrs[id(tensor)].append(name)
2646

2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
            # These are all the pointers of shared tensors
            if hasattr(self, "hf_device_map"):
                # if the model has offloaded parameters, we must check using find_tied_parameters()
                tied_params = find_tied_parameters(self)
                if tied_params:
                    tied_names = tied_params[0]
                    shared_ptrs = {
                        ptr: names for ptr, names in ptrs.items() if any(name in tied_names for name in names)
                    }
                else:
                    shared_ptrs = {}
            else:
                shared_ptrs = {ptr: names for ptr, names in ptrs.items() if len(names) > 1}

2661
2662
            # Recursively descend to find tied weight keys
            _tied_weights_keys = _get_tied_weight_keys(self)
2663
2664
            error_names = []
            to_delete_names = set()
2665
2666
2667
            for names in shared_ptrs.values():
                # Removing the keys which are declared as known duplicates on
                # load. This allows to make sure the name which is kept is consistent.
2668
                if _tied_weights_keys is not None:
2669
2670
                    found = 0
                    for name in sorted(names):
2671
                        matches_pattern = any(re.search(pat, name) for pat in _tied_weights_keys)
2672
                        if matches_pattern and name in state_dict:
2673
2674
                            found += 1
                            if found < len(names):
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
                                to_delete_names.add(name)
            # We are entering a place where the weights and the transformers configuration do NOT match.
            shared_names, disjoint_names = _find_disjoint(shared_ptrs.values(), state_dict)
            # Those are actually tensor sharing but disjoint from each other, we can safely clone them
            # Reloaded won't have the same property, but it shouldn't matter in any meaningful way.
            for name in disjoint_names:
                state_dict[name] = state_dict[name].clone()

            # When not all duplicates have been cleaned, still remove those keys, but put a clear warning.
            # If the link between tensors was done at runtime then `from_pretrained` will not get
            # the key back leading to random tensor. A proper warning will be shown
            # during reload (if applicable), but since the file is not necessarily compatible with
            # the config, better show a proper warning.
            shared_names, identical_names = _find_identical(shared_names, state_dict)
            # delete tensors that have identical storage
            for inames in identical_names:
                known = inames.intersection(to_delete_names)
                for name in known:
                    del state_dict[name]
                unknown = inames.difference(to_delete_names)
                if len(unknown) > 1:
                    error_names.append(unknown)

            if shared_names:
                error_names.append(set(shared_names))

            if len(error_names) > 0:
                raise RuntimeError(
                    f"The weights trying to be saved contained shared tensors {error_names} that are mismatching the transformers base configuration. Try saving using `safe_serialization=False` or remove this tensor sharing.",
2704
                )
2705

Sylvain Gugger's avatar
Sylvain Gugger committed
2706
        # Shard the model if it is too big.
2707
2708
2709
2710
2711
        if not _hf_peft_config_loaded:
            weights_name = SAFE_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME
            weights_name = _add_variant(weights_name, variant)
        else:
            weights_name = ADAPTER_SAFE_WEIGHTS_NAME if safe_serialization else ADAPTER_WEIGHTS_NAME
2712

2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
        state_dict_split = split_torch_state_dict_into_shards(
            state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size
        )
        # Save index if sharded
        index = None
        if state_dict_split.is_sharded:
            index = {
                "metadata": state_dict_split.metadata,
                "weight_map": state_dict_split.tensor_to_filename,
            }
Sylvain Gugger's avatar
Sylvain Gugger committed
2724
2725
2726
2727

        # Clean the folder from a previous save
        for filename in os.listdir(save_directory):
            full_filename = os.path.join(save_directory, filename)
2728
2729
            # If we have a shard file that is not going to be replaced, we delete it, but only from the main process
            # in distributed settings to avoid race conditions.
2730
            weights_no_suffix = weights_name.replace(".bin", "").replace(".safetensors", "")
2731
2732
2733

            # make sure that file to be deleted matches format of sharded file, e.g. pytorch_model-00001-of-00005
            filename_no_suffix = filename.replace(".bin", "").replace(".safetensors", "")
2734
            reg = re.compile(r"(.*?)-\d{5}-of-\d{5}")
2735

2736
            if (
2737
                filename.startswith(weights_no_suffix)
2738
                and os.path.isfile(full_filename)
2739
                and filename not in state_dict_split.filename_to_tensors.keys()
2740
                and is_main_process
2741
                and reg.fullmatch(filename_no_suffix) is not None
2742
            ):
Sylvain Gugger's avatar
Sylvain Gugger committed
2743
2744
                os.remove(full_filename)
        # Save the model
2745
2746
2747
2748
        filename_to_tensors = state_dict_split.filename_to_tensors.items()
        if module_map:
            filename_to_tensors = logging.tqdm(filename_to_tensors, desc="Saving checkpoint shards")
        for shard_file, tensors in filename_to_tensors:
2749
            shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors}
2750
2751
2752
2753
2754
2755
2756
2757
            # remake shard with onloaded parameters if necessary
            if module_map:
                if accelerate_version < version.parse("0.31"):
                    raise ImportError(
                        f"You need accelerate version to be greater or equal than 0.31 to save models with offloaded parameters. Detected version {accelerate_version}. "
                        f"Please upgrade accelerate with `pip install -U accelerate`"
                    )
                # init state_dict for this shard
2758
                shard_state_dict = {name: "" for name in shard}
2759
2760
2761
                for module_name in shard:
                    module = module_map[module_name]
                    # update state dict with onloaded parameters
2762
                    shard_state_dict = get_state_dict_from_offload(module, module_name, shard_state_dict)
2763
2764

                # assign shard to be the completed state dict
2765
2766
                shard = shard_state_dict
                del shard_state_dict
2767
2768
                gc.collect()

2769
2770
2771
2772
2773
2774
            if safe_serialization:
                # At some point we will need to deal better with save_function (used for TPU and other distributed
                # joyfulness), but for now this enough.
                safe_save_file(shard, os.path.join(save_directory, shard_file), metadata={"format": "pt"})
            else:
                save_function(shard, os.path.join(save_directory, shard_file))
Sylvain Gugger's avatar
Sylvain Gugger committed
2775
2776

        if index is None:
2777
            path_to_weights = os.path.join(save_directory, weights_name)
2778
            logger.info(f"Model weights saved in {path_to_weights}")
Sylvain Gugger's avatar
Sylvain Gugger committed
2779
        else:
2780
            save_index_file = SAFE_WEIGHTS_INDEX_NAME if safe_serialization else WEIGHTS_INDEX_NAME
2781
            save_index_file = os.path.join(save_directory, _add_variant(save_index_file, variant))
Sylvain Gugger's avatar
Sylvain Gugger committed
2782
2783
2784
2785
2786
2787
            # Save the index as well
            with open(save_index_file, "w", encoding="utf-8") as f:
                content = json.dumps(index, indent=2, sort_keys=True) + "\n"
                f.write(content)
            logger.info(
                f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be "
2788
                f"split in {len(state_dict_split.filename_to_tensors)} checkpoint shards. You can find where each parameters has been saved in the "
Sylvain Gugger's avatar
Sylvain Gugger committed
2789
2790
                f"index located at {save_index_file}."
            )
2791

Sylvain Gugger's avatar
Sylvain Gugger committed
2792
        if push_to_hub:
2793
2794
2795
2796
2797
2798
2799
2800
            # Eventually create an empty model card
            model_card = create_and_tag_model_card(
                repo_id, self.model_tags, token=token, ignore_metadata_errors=ignore_metadata_errors
            )

            # Update model card if needed:
            model_card.save(os.path.join(save_directory, "README.md"))

2801
            self._upload_modified_files(
2802
2803
2804
2805
                save_directory,
                repo_id,
                files_timestamps,
                commit_message=commit_message,
2806
                token=token,
2807
            )
Sylvain Gugger's avatar
Sylvain Gugger committed
2808

2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
    @wraps(PushToHubMixin.push_to_hub)
    def push_to_hub(self, *args, **kwargs):
        tags = self.model_tags if self.model_tags is not None else []

        tags_kwargs = kwargs.get("tags", [])
        if isinstance(tags_kwargs, str):
            tags_kwargs = [tags_kwargs]

        for tag in tags_kwargs:
            if tag not in tags:
                tags.append(tag)

        if tags:
            kwargs["tags"] = tags
        return super().push_to_hub(*args, **kwargs)

2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
    def get_memory_footprint(self, return_buffers=True):
        r"""
        Get the memory footprint of a model. This will return the memory footprint of the current model in bytes.
        Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the
        PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2

        Arguments:
            return_buffers (`bool`, *optional*, defaults to `True`):
                Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers
                are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch
                norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2
        """
        mem = sum([param.nelement() * param.element_size() for param in self.parameters()])
        if return_buffers:
            mem_bufs = sum([buf.nelement() * buf.element_size() for buf in self.buffers()])
            mem = mem + mem_bufs
        return mem

2843
    @wraps(torch.nn.Module.cuda)
2844
    def cuda(self, *args, **kwargs):
2845
2846
        if getattr(self, "quantization_method", None) == QuantizationMethod.HQQ:
            raise ValueError("`.cuda` is not supported for HQQ-quantized models.")
2847
        # Checks if the model has been loaded in 8-bit
Marc Sun's avatar
Marc Sun committed
2848
        if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
2849
2850
2851
2852
2853
2854
2855
            raise ValueError(
                "Calling `cuda()` is not supported for `4-bit` or `8-bit` quantized models. Please use the model as it is, since the"
                " model has already been set to the correct devices and casted to the correct `dtype`."
            )
        else:
            return super().cuda(*args, **kwargs)

2856
    @wraps(torch.nn.Module.to)
2857
    def to(self, *args, **kwargs):
2858
2859
        if getattr(self, "quantization_method", None) == QuantizationMethod.HQQ:
            raise ValueError("`.to` is not supported for HQQ-quantized models.")
2860
        # Checks if the model has been loaded in 8-bit
Marc Sun's avatar
Marc Sun committed
2861
        if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES:
2862
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2863
                "`.to` is not supported for `4-bit` or `8-bit` bitsandbytes models. Please use the model as it is, since the"
2864
2865
                " model has already been set to the correct devices and casted to the correct `dtype`."
            )
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
        elif getattr(self, "quantization_method", None) == QuantizationMethod.GPTQ:
            # For GPTQ models, we prevent users from casting the model to another dytpe to restrict unwanted behaviours.
            # the correct API should be to load the model with the desired dtype directly through `from_pretrained`.
            dtype_present_in_args = False

            if "dtype" not in kwargs:
                for arg in args:
                    if isinstance(arg, torch.dtype):
                        dtype_present_in_args = True
                        break
            else:
                dtype_present_in_args = True

            if dtype_present_in_args:
                raise ValueError(
                    "You cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired"
                    " `dtype` by passing the correct `torch_dtype` argument."
                )
        return super().to(*args, **kwargs)
2885
2886

    def half(self, *args):
Marc Sun's avatar
Marc Sun committed
2887
        # Checks if the model is quantized
2888
        if getattr(self, "is_quantized", False):
2889
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2890
                "`.half()` is not supported for quantized model. Please use the model as it is, since the"
2891
2892
2893
2894
2895
2896
                " model has already been casted to the correct `dtype`."
            )
        else:
            return super().half(*args)

    def float(self, *args):
Marc Sun's avatar
Marc Sun committed
2897
        # Checks if the model is quantized
2898
        if getattr(self, "is_quantized", False):
2899
            raise ValueError(
Marc Sun's avatar
Marc Sun committed
2900
                "`.float()` is not supported for quantized model. Please use the model as it is, since the"
2901
2902
2903
2904
2905
                " model has already been casted to the correct `dtype`."
            )
        else:
            return super().float(*args)

2906
    @classmethod
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
        *model_args,
        config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
        cache_dir: Optional[Union[str, os.PathLike]] = None,
        ignore_mismatched_sizes: bool = False,
        force_download: bool = False,
        local_files_only: bool = False,
        token: Optional[Union[str, bool]] = None,
        revision: str = "main",
        use_safetensors: bool = None,
        **kwargs,
2920
    ) -> "PreTrainedModel":
2921
2922
        r"""
        Instantiate a pretrained pytorch model from a pre-trained model configuration.
2923

Sylvain Gugger's avatar
Sylvain Gugger committed
2924
2925
        The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
        the model, you should first set it back in training mode with `model.train()`.
2926

2927
        The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come
2928
2929
        pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning
        task.
2930

2931
        The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those
2932
        weights are discarded.
2933

2934
2935
2936
2937
        If model weights are the same precision as the base model (and is a supported model), weights will be lazily loaded
        in using the `meta` device and brought into memory once an input is passed through that layer regardless of
        `low_cpu_mem_usage`.

2938
        Parameters:
2939
            pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
2940
2941
                Can be either:

2942
2943
2944
2945
                    - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
                    - A path to a *directory* containing model weights saved using
                      [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
                    - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
Sylvain Gugger's avatar
Sylvain Gugger committed
2946
2947
2948
                      this case, `from_tf` should be set to `True` and a configuration object should be provided as
                      `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
                      PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
2949
                    - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g,
Sylvain Gugger's avatar
Sylvain Gugger committed
2950
2951
                      `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to
                      `True`.
2952
2953
2954
2955
2956
                    - `None` if you are both providing the configuration and state dictionary (resp. with keyword
                      arguments `config` and `state_dict`).
            model_args (sequence of positional arguments, *optional*):
                All remaining positional arguments will be passed to the underlying model's `__init__` method.
            config (`Union[PretrainedConfig, str, os.PathLike]`, *optional*):
2957
2958
                Can be either:

2959
2960
                    - an instance of a class derived from [`PretrainedConfig`],
                    - a string or path valid as input to [`~PretrainedConfig.from_pretrained`].
2961

2962
                Configuration for the model to use instead of an automatically loaded configuration. Configuration can
2963
2964
                be automatically loaded when:

2965
                    - The model is a model provided by the library (loaded with the *model id* string of a pretrained
2966
                      model).
Sylvain Gugger's avatar
Sylvain Gugger committed
2967
2968
                    - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
                      save directory.
2969
2970
2971
                    - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
                      configuration JSON file named *config.json* is found in the directory.
            state_dict (`Dict[str, torch.Tensor]`, *optional*):
2972
2973
2974
                A state dictionary to use instead of a state dictionary loaded from saved weights file.

                This option can be used if you want to create a model from a pretrained configuration but load your own
Sylvain Gugger's avatar
Sylvain Gugger committed
2975
                weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and
2976
2977
                [`~PreTrainedModel.from_pretrained`] is not a simpler option.
            cache_dir (`Union[str, os.PathLike]`, *optional*):
2978
2979
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
2980
            from_tf (`bool`, *optional*, defaults to `False`):
2981
                Load the model weights from a TensorFlow checkpoint save file (see docstring of
2982
2983
                `pretrained_model_name_or_path` argument).
            from_flax (`bool`, *optional*, defaults to `False`):
2984
                Load the model weights from a Flax checkpoint save file (see docstring of
2985
2986
                `pretrained_model_name_or_path` argument).
            ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`):
2987
2988
2989
                Whether or not to raise an error if some of the weights from the checkpoint do not have the same size
                as the weights of the model (if for instance, you are instantiating a model with 10 labels from a
                checkpoint with 3 labels).
2990
            force_download (`bool`, *optional*, defaults to `False`):
2991
2992
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
2993
2994
2995
            resume_download:
                Deprecated and ignored. All downloads are now resumed by default when possible.
                Will be removed in v5 of Transformers.
2996
            proxies (`Dict[str, str]`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
2997
2998
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
2999
            output_loading_info(`bool`, *optional*, defaults to `False`):
Sylvain Gugger's avatar
Sylvain Gugger committed
3000
                Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
3001
            local_files_only(`bool`, *optional*, defaults to `False`):
Stas Bekman's avatar
Stas Bekman committed
3002
                Whether or not to only look at local files (i.e., do not try to download the model).
3003
            token (`str` or `bool`, *optional*):
3004
3005
                The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use
                the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).
3006
            revision (`str`, *optional*, defaults to `"main"`):
Julien Chaumond's avatar
Julien Chaumond committed
3007
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
3008
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
Julien Chaumond's avatar
Julien Chaumond committed
3009
                identifier allowed by git.
3010
3011
3012
3013
3014
3015
3016

                <Tip>

                To test a pull request you made on the Hub, you can pass `revision="refs/pr/<pr_number>".

                </Tip>

3017
            mirror (`str`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
3018
3019
3020
                Mirror source to accelerate downloads in China. If you are from China and have an accessibility
                problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety.
                Please refer to the mirror site for more information.
3021
            _fast_init(`bool`, *optional*, defaults to `True`):
3022
3023
                Whether or not to disable fast initialization.

3024
3025
                <Tip warning={true}>

Sylvain Gugger's avatar
Sylvain Gugger committed
3026
3027
3028
                One should only disable *_fast_init* to ensure backwards compatibility with `transformers.__version__ <
                4.6.0` for seeded model initialization. This argument will be removed at the next major version. See
                [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information.
3029

3030
                </Tip>
3031
3032
            attn_implementation (`str`, *optional*):
                The attention implementation to use in the model (if relevant). Can be any of `"eager"` (manual implementation of the attention), `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)), or `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)). By default, if available, SDPA will be used for torch>=2.1.1. The default is otherwise the manual `"eager"` implementation.
3033

3034
3035
3036
            > Parameters for big model inference

            low_cpu_mem_usage(`bool`, *optional*):
3037
                Tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
3038
                Generally should be combined with a `device_map` (such as `"auto"`) for best results.
3039
                This is an experimental feature and a subject to change at any moment.
3040
3041
3042
3043
3044
                </Tip>
                    If the model weights are in the same precision as the model loaded in, `low_cpu_mem_usage` (without
                    `device_map`) is redundant and will not provide any benefit in regards to CPU memory usage. However,
                    this should still be enabled if you are passing in a `device_map`.
                </Tip>
3045
            torch_dtype (`str` or `torch.dtype`, *optional*):
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
                Override the default `torch.dtype` and load the model under a specific `dtype`. The different options
                are:

                1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified
                  `dtype`, ignoring the model's `config.torch_dtype` if one exists. If not specified
                  - the model will get loaded in `torch.float` (fp32).

                2. `"auto"` - A `torch_dtype` entry in the `config.json` file of the model will be
                  attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in
                  the checkpoint that's of a floating point type and use that as `dtype`. This will load the model
                  using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how
                  the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32.

3059
3060
                3. A string that is a valid `torch.dtype`. E.g. "float32" loads the model in `torch.float32`, "float16" loads in `torch.float16` etc.

3061
3062
3063
3064
3065
3066
3067
3068
                <Tip>

                For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or
                reach out to the authors and ask them to add this information to the model's card and to insert the
                `torch_dtype` entry in `config.json` on the hub.

                </Tip>

3069
            device_map (`str` or `Dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*):
3070
3071
                A map that specifies where each submodule should go. It doesn't need to be refined to each
                parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the
3072
3073
3074
                same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank
                like `1`) on which the model will be allocated, the device map will map the entire model to this
                device. Passing `device_map = 0` means put the whole model on GPU 0.
3075

3076
3077
                To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For
                more information about each option see [designing a device
Patrick von Platen's avatar
Patrick von Platen committed
3078
                map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).
3079
3080
3081
            max_memory (`Dict`, *optional*):
                A dictionary device identifier to maximum memory. Will default to the maximum memory available for each
                GPU and the available CPU RAM if unset.
3082
3083
            offload_folder (`str` or `os.PathLike`, *optional*):
                If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
3084
            offload_state_dict (`bool`, *optional*):
3085
                If `True`, will temporarily offload the CPU state dict to the hard drive to avoid getting out of CPU
3086
3087
                RAM if the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to
                `True` when there is some disk offload.
3088
3089
            offload_buffers (`bool`, *optional*):
                Whether or not to offload the buffers with the model parameters.
Marc Sun's avatar
Marc Sun committed
3090
3091
            quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*):
                A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g
3092
3093
3094
3095
                bitsandbytes, gptq). There may be other quantization-related kwargs, including `load_in_4bit` and
                `load_in_8bit`, which are parsed by QuantizationConfigParser. Supported only for bitsandbytes
                quantizations and not preferred. consider inserting all such arguments into quantization_config
                instead.
3096
3097
3098
            subfolder (`str`, *optional*, defaults to `""`):
                In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
                specify the folder name here.
3099
3100
3101
            variant (`str`, *optional*):
                If specified load weights from `variant` filename, *e.g.* pytorch_model.<variant>.bin. `variant` is
                ignored when using `from_tf` or `from_flax`.
3102
3103
3104
            use_safetensors (`bool`, *optional*, defaults to `None`):
                Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors`
                is not installed, it will be set to `False`.
3105

3106
            kwargs (remaining dictionary of keyword arguments, *optional*):
3107
                Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
3108
                `output_attentions=True`). Behaves differently depending on whether a `config` is provided or
3109
3110
                automatically loaded:

3111
3112
                    - If a configuration is provided with `config`, `**kwargs` will be directly passed to the
                      underlying model's `__init__` method (we assume all relevant updates to the configuration have
3113
                      already been done)
3114
                    - If a configuration is not provided, `kwargs` will be first passed to the configuration class
Sylvain Gugger's avatar
Sylvain Gugger committed
3115
3116
3117
3118
                      initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
                      corresponds to a configuration attribute will be used to override said attribute with the
                      supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
                      will be passed to the underlying model's `__init__` function.
3119
3120
3121

        <Tip>

Sylvain Gugger's avatar
Sylvain Gugger committed
3122
3123
        Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to
        use this method in a firewalled environment.
3124
3125
3126
3127
3128
3129
3130

        </Tip>

        Examples:

        ```python
        >>> from transformers import BertConfig, BertModel
Sylvain Gugger's avatar
Sylvain Gugger committed
3131

3132
        >>> # Download model and configuration from huggingface.co and cache.
3133
        >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased")
3134
        >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable).
Sylvain Gugger's avatar
Sylvain Gugger committed
3135
        >>> model = BertModel.from_pretrained("./test/saved_model/")
3136
        >>> # Update configuration during loading.
3137
        >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", output_attentions=True)
3138
3139
        >>> assert model.config.output_attentions == True
        >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable).
Sylvain Gugger's avatar
Sylvain Gugger committed
3140
3141
        >>> config = BertConfig.from_json_file("./tf_model/my_tf_model_config.json")
        >>> model = BertModel.from_pretrained("./tf_model/my_tf_checkpoint.ckpt.index", from_tf=True, config=config)
3142
        >>> # Loading from a Flax checkpoint file instead of a PyTorch model (slower)
3143
        >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", from_flax=True)
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
        ```

        * `low_cpu_mem_usage` algorithm:

        This is an experimental function that loads the model using ~1x model size CPU memory

        Here is how it works:

        1. save which state_dict keys we have
        2. drop state_dict before the model is created, since the latter takes 1x model size CPU memory
        3. after the model has been instantiated switch to the meta device all params/buffers that
        are going to be replaced from the loaded state_dict
        4. load state_dict 2nd time
        5. replace the params/buffers from the state_dict

        Currently, it can't handle deepspeed ZeRO stage 3 and ignores loading errors

        """
3162
3163
        state_dict = kwargs.pop("state_dict", None)
        from_tf = kwargs.pop("from_tf", False)
3164
        from_flax = kwargs.pop("from_flax", False)
3165
        resume_download = kwargs.pop("resume_download", None)
3166
3167
        proxies = kwargs.pop("proxies", None)
        output_loading_info = kwargs.pop("output_loading_info", False)
3168
        use_auth_token = kwargs.pop("use_auth_token", None)
3169
        trust_remote_code = kwargs.pop("trust_remote_code", None)
Sylvain Gugger's avatar
Sylvain Gugger committed
3170
        _ = kwargs.pop("mirror", None)
3171
3172
        from_pipeline = kwargs.pop("_from_pipeline", None)
        from_auto_class = kwargs.pop("_from_auto", False)
3173
        _fast_init = kwargs.pop("_fast_init", True)
3174
        torch_dtype = kwargs.pop("torch_dtype", None)
3175
3176
        low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", None)
        device_map = kwargs.pop("device_map", None)
3177
        max_memory = kwargs.pop("max_memory", None)
3178
        offload_folder = kwargs.pop("offload_folder", None)
3179
        offload_state_dict = kwargs.pop("offload_state_dict", False)
3180
        offload_buffers = kwargs.pop("offload_buffers", False)
3181
        load_in_8bit = kwargs.pop("load_in_8bit", False)
3182
        load_in_4bit = kwargs.pop("load_in_4bit", False)
3183
        quantization_config = kwargs.pop("quantization_config", None)
3184
        subfolder = kwargs.pop("subfolder", "")
3185
        commit_hash = kwargs.pop("_commit_hash", None)
3186
        variant = kwargs.pop("variant", None)
3187
        adapter_kwargs = kwargs.pop("adapter_kwargs", {})
3188
        adapter_name = kwargs.pop("adapter_name", "default")
3189
        use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False)
3190

3191
3192
3193
3194
        gguf_file = kwargs.pop("gguf_file", None)
        # Cache path to the GGUF file
        gguf_path = None

3195
3196
3197
        if is_fsdp_enabled():
            low_cpu_mem_usage = True

3198
3199
        if use_auth_token is not None:
            warnings.warn(
3200
3201
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
3202
3203
3204
3205
3206
3207
3208
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

3209
        if token is not None and adapter_kwargs is not None and "token" not in adapter_kwargs:
3210
3211
            adapter_kwargs["token"] = token

3212
3213
        if use_safetensors is None and not is_safetensors_available():
            use_safetensors = False
3214
3215
3216
3217
3218
        if trust_remote_code is True:
            logger.warning(
                "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is"
                " ignored."
            )
3219

3220
3221
3222
        if gguf_file is not None and not is_accelerate_available():
            raise ValueError("accelerate is required when loading a GGUF file `pip install accelerate`.")

3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
        if commit_hash is None:
            if not isinstance(config, PretrainedConfig):
                # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible
                resolved_config_file = cached_file(
                    pretrained_model_name_or_path,
                    CONFIG_NAME,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
                    token=token,
                    revision=revision,
                    subfolder=subfolder,
3237
                    _raise_exceptions_for_gated_repo=False,
3238
3239
3240
3241
3242
3243
3244
                    _raise_exceptions_for_missing_entries=False,
                    _raise_exceptions_for_connection_errors=False,
                )
                commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
            else:
                commit_hash = getattr(config, "_commit_hash", None)

3245
        if is_peft_available():
3246
3247
            _adapter_model_path = adapter_kwargs.pop("_adapter_model_path", None)

3248
3249
3250
3251
3252
3253
3254
3255
3256
            if _adapter_model_path is None:
                _adapter_model_path = find_adapter_config_file(
                    pretrained_model_name_or_path,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
                    _commit_hash=commit_hash,
3257
                    **adapter_kwargs,
3258
3259
                )
            if _adapter_model_path is not None and os.path.isfile(_adapter_model_path):
3260
                with open(_adapter_model_path, "r", encoding="utf-8") as f:
3261
                    _adapter_model_path = pretrained_model_name_or_path
3262
                    pretrained_model_name_or_path = json.load(f)["base_model_name_or_path"]
3263
3264
        else:
            _adapter_model_path = None
3265

3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
        # change device_map into a map if we passed an int, a str or a torch.device
        if isinstance(device_map, torch.device):
            device_map = {"": device_map}
        elif isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
            try:
                device_map = {"": torch.device(device_map)}
            except RuntimeError:
                raise ValueError(
                    "When passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or "
                    f"'auto', 'balanced', 'balanced_low_0', 'sequential' but found {device_map}."
                )
        elif isinstance(device_map, int):
            if device_map < 0:
                raise ValueError(
                    "You can't pass device_map as a negative int. If you want to put the model on the cpu, pass device_map = 'cpu' "
                )
            else:
                device_map = {"": device_map}

3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
        if device_map is not None:
            if low_cpu_mem_usage is None:
                low_cpu_mem_usage = True
            elif not low_cpu_mem_usage:
                raise ValueError("Passing along a `device_map` requires `low_cpu_mem_usage=True`")

        if low_cpu_mem_usage:
            if is_deepspeed_zero3_enabled():
                raise ValueError(
                    "DeepSpeed Zero-3 is not compatible with `low_cpu_mem_usage=True` or with passing a `device_map`."
                )
            elif not is_accelerate_available():
                raise ImportError(
3298
                    f"Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install 'accelerate>={ACCELERATE_MIN_VERSION}'`"
3299
                )
3300

3301
3302
3303
        # handling bnb config from kwargs, remove after `load_in_{4/8}bit` deprecation.
        if load_in_4bit or load_in_8bit:
            if quantization_config is not None:
3304
                raise ValueError(
3305
                    "You can't pass `load_in_4bit`or `load_in_8bit` as a kwarg when passing "
3306
3307
3308
                    "`quantization_config` argument at the same time."
                )

3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
            # preparing BitsAndBytesConfig from kwargs
            config_dict = {k: v for k, v in kwargs.items() if k in inspect.signature(BitsAndBytesConfig).parameters}
            config_dict = {**config_dict, "load_in_4bit": load_in_4bit, "load_in_8bit": load_in_8bit}
            quantization_config, kwargs = BitsAndBytesConfig.from_dict(
                config_dict=config_dict, return_unused_kwargs=True, **kwargs
            )
            logger.warning(
                "The `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. "
                "Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead."
            )
3319

3320
        from_pt = not (from_tf | from_flax)
3321

3322
3323
3324
        user_agent = {"file_type": "model", "framework": "pytorch", "from_auto_class": from_auto_class}
        if from_pipeline is not None:
            user_agent["using_pipeline"] = from_pipeline
thomwolf's avatar
thomwolf committed
3325

3326
3327
3328
3329
        if is_offline_mode() and not local_files_only:
            logger.info("Offline mode: forcing local_files_only=True")
            local_files_only = True

3330
3331
3332
        # Load config if we don't provide a configuration
        if not isinstance(config, PretrainedConfig):
            config_path = config if config is not None else pretrained_model_name_or_path
3333
            config, model_kwargs = cls.config_class.from_pretrained(
3334
3335
3336
                config_path,
                cache_dir=cache_dir,
                return_unused_kwargs=True,
3337
                force_download=force_download,
3338
                resume_download=resume_download,
3339
                proxies=proxies,
3340
                local_files_only=local_files_only,
3341
                token=token,
Julien Chaumond's avatar
Julien Chaumond committed
3342
                revision=revision,
3343
                subfolder=subfolder,
3344
3345
                _from_auto=from_auto_class,
                _from_pipeline=from_pipeline,
3346
                **kwargs,
3347
3348
            )
        else:
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
            # In case one passes a config to `from_pretrained` + "attn_implementation"
            # override the `_attn_implementation` attribute to `attn_implementation` of the kwargs
            # Please see: https://github.com/huggingface/transformers/issues/28038

            # Overwrite `config._attn_implementation` by the one from the kwargs --> in auto-factory
            # we pop attn_implementation from the kwargs but this handles the case where users
            # passes manually the config to `from_pretrained`.
            config = copy.deepcopy(config)

            kwarg_attn_imp = kwargs.pop("attn_implementation", None)
3359
            if kwarg_attn_imp is not None:
3360
                config._attn_implementation = kwarg_attn_imp
3361

3362
            model_kwargs = kwargs
3363

3364
3365
3366
3367
3368
        pre_quantized = getattr(config, "quantization_config", None) is not None
        if pre_quantized or quantization_config is not None:
            if pre_quantized:
                config.quantization_config = AutoHfQuantizer.merge_quantization_configs(
                    config.quantization_config, quantization_config
Marc Sun's avatar
Marc Sun committed
3369
3370
3371
                )
            else:
                config.quantization_config = quantization_config
3372
3373
3374
            hf_quantizer = AutoHfQuantizer.from_config(config.quantization_config, pre_quantized=pre_quantized)
        else:
            hf_quantizer = None
3375

3376
3377
3378
3379
3380
3381
        if hf_quantizer is not None:
            hf_quantizer.validate_environment(
                torch_dtype=torch_dtype, from_tf=from_tf, from_flax=from_flax, device_map=device_map
            )
            torch_dtype = hf_quantizer.update_torch_dtype(torch_dtype)
            device_map = hf_quantizer.update_device_map(device_map)
3382

3383
3384
3385
            # In order to ensure popular quantization methods are supported. Can be disable with `disable_telemetry`
            user_agent["quant"] = hf_quantizer.quantization_config.quant_method.value

3386
3387
3388
            # Force-set to `True` for more mem efficiency
            if low_cpu_mem_usage is None:
                low_cpu_mem_usage = True
3389
                logger.warning("`low_cpu_mem_usage` was None, now set to True since model is quantized.")
3390
        is_quantized = hf_quantizer is not None
3391

Sylvain Gugger's avatar
Sylvain Gugger committed
3392
3393
3394
3395
        # This variable will flag if we're loading a sharded checkpoint. In this case the archive file is just the
        # index of the files.
        is_sharded = False
        sharded_metadata = None
thomwolf's avatar
thomwolf committed
3396
        # Load model
Yih-Dar's avatar
Yih-Dar committed
3397
3398
        loading_info = None

3399
3400
3401
3402
        # Keep in fp32 modules
        keep_in_fp32_modules = None
        use_keep_in_fp32_modules = False

3403
3404
3405
3406
3407
3408
        if gguf_file is not None and hf_quantizer is not None:
            raise ValueError(
                "You cannot combine Quantization and loading a model from a GGUF file, try again by making sure you did not passed a `quantization_config` or that you did not load a quantized model from the Hub."
            )

        if pretrained_model_name_or_path is not None and gguf_file is None:
3409
            pretrained_model_name_or_path = str(pretrained_model_name_or_path)
3410
3411
            is_local = os.path.isdir(pretrained_model_name_or_path)
            if is_local:
3412
3413
3414
                if from_tf and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                ):
3415
                    # Load from a TF 1.0 checkpoint in priority if from_tf
3416
3417
3418
3419
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index")
                elif from_tf and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)
                ):
3420
                    # Load from a TF 2.0 checkpoint in priority if from_tf
3421
3422
3423
3424
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME)
                elif from_flax and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
                ):
3425
                    # Load from a Flax checkpoint in priority if from_flax
3426
                    archive_file = os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
3427
                elif use_safetensors is not False and os.path.isfile(
3428
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant))
3429
3430
                ):
                    # Load from a safetensors checkpoint
3431
3432
3433
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant)
                    )
3434
                elif use_safetensors is not False and os.path.isfile(
3435
3436
3437
                    os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)
                    )
3438
3439
                ):
                    # Load from a sharded safetensors checkpoint
3440
3441
3442
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)
                    )
3443
                    is_sharded = True
3444
                elif not use_safetensors and os.path.isfile(
3445
3446
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_NAME, variant))
                ):
thomwolf's avatar
thomwolf committed
3447
                    # Load from a PyTorch checkpoint
3448
3449
3450
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_NAME, variant)
                    )
3451
                elif not use_safetensors and os.path.isfile(
3452
3453
                    os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_INDEX_NAME, variant))
                ):
Sylvain Gugger's avatar
Sylvain Gugger committed
3454
                    # Load from a sharded PyTorch checkpoint
3455
3456
3457
                    archive_file = os.path.join(
                        pretrained_model_name_or_path, subfolder, _add_variant(WEIGHTS_INDEX_NAME, variant)
                    )
Sylvain Gugger's avatar
Sylvain Gugger committed
3458
                    is_sharded = True
3459
                # At this stage we don't have a weight file so we will raise an error.
3460
3461
3462
3463
                elif not use_safetensors and (
                    os.path.isfile(os.path.join(pretrained_model_name_or_path, subfolder, TF_WEIGHTS_NAME + ".index"))
                    or os.path.isfile(os.path.join(pretrained_model_name_or_path, subfolder, TF2_WEIGHTS_NAME))
                ):
3464
                    raise EnvironmentError(
3465
3466
3467
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path} but there is a file for TensorFlow weights. Use"
                        " `from_tf=True` to load this model from those weights."
3468
                    )
3469
3470
3471
                elif not use_safetensors and os.path.isfile(
                    os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME)
                ):
3472
                    raise EnvironmentError(
3473
3474
3475
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path} but there is a file for Flax weights. Use `from_flax=True`"
                        " to load this model from those weights."
3476
                    )
3477
3478
3479
3480
3481
                elif use_safetensors:
                    raise EnvironmentError(
                        f"Error no file named {_add_variant(SAFE_WEIGHTS_NAME, variant)} found in directory"
                        f" {pretrained_model_name_or_path}."
                    )
thomwolf's avatar
thomwolf committed
3482
                else:
3483
                    raise EnvironmentError(
3484
3485
                        f"Error no file named {_add_variant(WEIGHTS_NAME, variant)}, {_add_variant(SAFE_WEIGHTS_NAME, variant)},"
                        f" {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME + '.index'} or {FLAX_WEIGHTS_NAME} found in directory"
3486
                        f" {pretrained_model_name_or_path}."
3487
                    )
3488
            elif os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path)):
3489
                archive_file = pretrained_model_name_or_path
3490
                is_local = True
3491
            elif os.path.isfile(os.path.join(subfolder, pretrained_model_name_or_path + ".index")):
3492
3493
3494
3495
3496
                if not from_tf:
                    raise ValueError(
                        f"We found a TensorFlow checkpoint at {pretrained_model_name_or_path + '.index'}, please set "
                        "from_tf to True to load from this checkpoint."
                    )
3497
                archive_file = os.path.join(subfolder, pretrained_model_name_or_path + ".index")
3498
                is_local = True
3499
            elif is_remote_url(pretrained_model_name_or_path):
3500
                filename = pretrained_model_name_or_path
3501
                resolved_archive_file = download_url(pretrained_model_name_or_path)
3502
            else:
3503
3504
3505
3506
3507
                # set correct filename
                if from_tf:
                    filename = TF2_WEIGHTS_NAME
                elif from_flax:
                    filename = FLAX_WEIGHTS_NAME
3508
                elif use_safetensors is not False:
3509
                    filename = _add_variant(SAFE_WEIGHTS_NAME, variant)
3510
                else:
3511
                    filename = _add_variant(WEIGHTS_NAME, variant)
3512

3513
3514
                try:
                    # Load from URL or cache if already cached
3515
3516
3517
3518
3519
3520
                    cached_file_kwargs = {
                        "cache_dir": cache_dir,
                        "force_download": force_download,
                        "proxies": proxies,
                        "resume_download": resume_download,
                        "local_files_only": local_files_only,
3521
                        "token": token,
3522
3523
3524
                        "user_agent": user_agent,
                        "revision": revision,
                        "subfolder": subfolder,
3525
                        "_raise_exceptions_for_gated_repo": False,
3526
3527
3528
                        "_raise_exceptions_for_missing_entries": False,
                        "_commit_hash": commit_hash,
                    }
3529
                    resolved_archive_file = cached_file(pretrained_model_name_or_path, filename, **cached_file_kwargs)
3530

3531
                    # Since we set _raise_exceptions_for_missing_entries=False, we don't get an exception but a None
3532
                    # result when internet is up, the repo and revision exist, but the file does not.
3533
                    if resolved_archive_file is None and filename == _add_variant(SAFE_WEIGHTS_NAME, variant):
3534
3535
                        # Maybe the checkpoint is sharded, we try to grab the index name in this case.
                        resolved_archive_file = cached_file(
3536
3537
3538
                            pretrained_model_name_or_path,
                            _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant),
                            **cached_file_kwargs,
3539
3540
3541
                        )
                        if resolved_archive_file is not None:
                            is_sharded = True
3542
                        elif use_safetensors:
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
                            if revision == "main":
                                resolved_archive_file, revision, is_sharded = auto_conversion(
                                    pretrained_model_name_or_path, **cached_file_kwargs
                                )
                            cached_file_kwargs["revision"] = revision
                            if resolved_archive_file is None:
                                raise EnvironmentError(
                                    f"{pretrained_model_name_or_path} does not appear to have a file named"
                                    f" {_add_variant(SAFE_WEIGHTS_NAME, variant)} or {_add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)} "
                                    "and thus cannot be loaded with `safetensors`. Please make sure that the model has "
                                    "been saved with `safe_serialization=True` or do not set `use_safetensors=True`."
                                )
3555
3556
                        else:
                            # This repo has no safetensors file of any kind, we switch to PyTorch.
3557
                            filename = _add_variant(WEIGHTS_NAME, variant)
3558
                            resolved_archive_file = cached_file(
3559
                                pretrained_model_name_or_path, filename, **cached_file_kwargs
3560
                            )
3561
                    if resolved_archive_file is None and filename == _add_variant(WEIGHTS_NAME, variant):
Sylvain Gugger's avatar
Sylvain Gugger committed
3562
                        # Maybe the checkpoint is sharded, we try to grab the index name in this case.
3563
                        resolved_archive_file = cached_file(
3564
3565
3566
                            pretrained_model_name_or_path,
                            _add_variant(WEIGHTS_INDEX_NAME, variant),
                            **cached_file_kwargs,
3567
                        )
3568
3569
                        if resolved_archive_file is not None:
                            is_sharded = True
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
                    if not local_files_only and not is_offline_mode():
                        if resolved_archive_file is not None:
                            if filename in [WEIGHTS_NAME, WEIGHTS_INDEX_NAME]:
                                # If the PyTorch file was found, check if there is a safetensors file on the repository
                                # If there is no safetensors file on the repositories, start an auto conversion
                                safe_weights_name = SAFE_WEIGHTS_INDEX_NAME if is_sharded else SAFE_WEIGHTS_NAME
                                has_file_kwargs = {
                                    "revision": revision,
                                    "proxies": proxies,
                                    "token": token,
3580
3581
                                    "cache_dir": cache_dir,
                                    "local_files_only": local_files_only,
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
                                }
                                cached_file_kwargs = {
                                    "cache_dir": cache_dir,
                                    "force_download": force_download,
                                    "resume_download": resume_download,
                                    "local_files_only": local_files_only,
                                    "user_agent": user_agent,
                                    "subfolder": subfolder,
                                    "_raise_exceptions_for_gated_repo": False,
                                    "_raise_exceptions_for_missing_entries": False,
                                    "_commit_hash": commit_hash,
                                    **has_file_kwargs,
                                }
                                if not has_file(pretrained_model_name_or_path, safe_weights_name, **has_file_kwargs):
                                    Thread(
                                        target=auto_conversion,
                                        args=(pretrained_model_name_or_path,),
                                        kwargs={"ignore_errors_during_conversion": True, **cached_file_kwargs},
                                        name="Thread-autoconversion",
                                    ).start()
                        else:
                            # Otherwise, no PyTorch file was found, maybe there is a TF or Flax model file.
                            # We try those to give a helpful error message.
3605
3606
3607
3608
                            has_file_kwargs = {
                                "revision": revision,
                                "proxies": proxies,
                                "token": token,
3609
3610
                                "cache_dir": cache_dir,
                                "local_files_only": local_files_only,
3611
                            }
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
                            if has_file(pretrained_model_name_or_path, TF2_WEIGHTS_NAME, **has_file_kwargs):
                                raise EnvironmentError(
                                    f"{pretrained_model_name_or_path} does not appear to have a file named"
                                    f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file for TensorFlow weights."
                                    " Use `from_tf=True` to load this model from those weights."
                                )
                            elif has_file(pretrained_model_name_or_path, FLAX_WEIGHTS_NAME, **has_file_kwargs):
                                raise EnvironmentError(
                                    f"{pretrained_model_name_or_path} does not appear to have a file named"
                                    f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file for Flax weights. Use"
                                    " `from_flax=True` to load this model from those weights."
                                )
                            elif variant is not None and has_file(
                                pretrained_model_name_or_path, WEIGHTS_NAME, **has_file_kwargs
                            ):
                                raise EnvironmentError(
                                    f"{pretrained_model_name_or_path} does not appear to have a file named"
                                    f" {_add_variant(WEIGHTS_NAME, variant)} but there is a file without the variant"
                                    f" {variant}. Use `variant=None` to load this model from those weights."
                                )
                            else:
                                raise EnvironmentError(
                                    f"{pretrained_model_name_or_path} does not appear to have a file named"
                                    f" {_add_variant(WEIGHTS_NAME, variant)}, {_add_variant(SAFE_WEIGHTS_NAME, variant)},"
                                    f" {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME} or {FLAX_WEIGHTS_NAME}."
                                )
3638

3639
3640
3641
3642
                except EnvironmentError:
                    # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted
                    # to the original exception.
                    raise
3643
                except Exception as e:
3644
                    # For any other exception, we throw a generic error.
3645
                    raise EnvironmentError(
3646
3647
3648
                        f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it"
                        " from 'https://huggingface.co/models', make sure you don't have a local directory with the"
                        f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
3649
3650
                        f" directory containing a file named {_add_variant(WEIGHTS_NAME, variant)},"
                        f" {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME} or {FLAX_WEIGHTS_NAME}."
3651
                    ) from e
3652

3653
            if is_local:
3654
                logger.info(f"loading weights file {archive_file}")
3655
                resolved_archive_file = archive_file
3656
            else:
3657
                logger.info(f"loading weights file {filename} from cache at {resolved_archive_file}")
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
        elif gguf_file:
            from .modeling_gguf_pytorch_utils import load_gguf_checkpoint

            # Case 1: the GGUF file is present locally
            if os.path.isfile(gguf_file):
                gguf_path = gguf_file
            # Case 2: The GGUF path is a location on the Hub
            # Load from URL or cache if already cached
            else:
                cached_file_kwargs = {
                    "cache_dir": cache_dir,
                    "force_download": force_download,
                    "proxies": proxies,
                    "resume_download": resume_download,
                    "local_files_only": local_files_only,
                    "token": token,
                    "user_agent": user_agent,
                    "revision": revision,
                    "subfolder": subfolder,
                    "_raise_exceptions_for_gated_repo": False,
                    "_raise_exceptions_for_missing_entries": False,
                    "_commit_hash": commit_hash,
                }

                gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **cached_file_kwargs)

            state_dict = load_gguf_checkpoint(gguf_path, return_tensors=True)["tensors"]

            resolved_archive_file = None
            is_sharded = False
3688
        else:
thomwolf's avatar
thomwolf committed
3689
            resolved_archive_file = None
3690

Sylvain Gugger's avatar
Sylvain Gugger committed
3691
3692
        # We'll need to download and cache each checkpoint shard if the checkpoint is sharded.
        if is_sharded:
3693
            # resolved_archive_file becomes a list of files that point to the different checkpoint shards in this case.
Sylvain Gugger's avatar
Sylvain Gugger committed
3694
3695
3696
3697
3698
3699
3700
3701
            resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
                pretrained_model_name_or_path,
                resolved_archive_file,
                cache_dir=cache_dir,
                force_download=force_download,
                proxies=proxies,
                resume_download=resume_download,
                local_files_only=local_files_only,
3702
                token=token,
Sylvain Gugger's avatar
Sylvain Gugger committed
3703
3704
                user_agent=user_agent,
                revision=revision,
3705
                subfolder=subfolder,
3706
                _commit_hash=commit_hash,
Sylvain Gugger's avatar
Sylvain Gugger committed
3707
3708
            )

3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
        if (
            is_safetensors_available()
            and isinstance(resolved_archive_file, str)
            and resolved_archive_file.endswith(".safetensors")
        ):
            with safe_open(resolved_archive_file, framework="pt") as f:
                metadata = f.metadata()

            if metadata.get("format") == "pt":
                pass
            elif metadata.get("format") == "tf":
                from_tf = True
                logger.info("A TensorFlow safetensors file is being loaded in a PyTorch model.")
            elif metadata.get("format") == "flax":
                from_flax = True
                logger.info("A Flax safetensors file is being loaded in a PyTorch model.")
3725
3726
3727
            elif metadata.get("format") == "mlx":
                # This is a mlx file, we assume weights are compatible with pt
                pass
3728
3729
            else:
                raise ValueError(
3730
                    f"Incompatible safetensors file. File metadata is not ['pt', 'tf', 'flax', 'mlx'] but {metadata.get('format')}"
3731
3732
3733
3734
                )

        from_pt = not (from_tf | from_flax)

3735
3736
        # load pt weights early so that we know which dtype to init the model under
        if from_pt:
3737
            if not is_sharded and state_dict is None:
Sylvain Gugger's avatar
Sylvain Gugger committed
3738
3739
                # Time to load the checkpoint
                state_dict = load_state_dict(resolved_archive_file)
3740

3741
3742
3743
            # set dtype to instantiate the model under:
            # 1. If torch_dtype is not None, we use that dtype
            # 2. If torch_dtype is "auto", we auto-detect dtype from the loaded state_dict, by checking its first
3744
            #    weights entry that is of a floating type - we assume all floating dtype weights are of the same dtype
3745
3746
            # we also may have config.torch_dtype available, but we won't rely on it till v5
            dtype_orig = None
3747

3748
3749
3750
            if torch_dtype is not None:
                if isinstance(torch_dtype, str):
                    if torch_dtype == "auto":
3751
3752
3753
                        if hasattr(config, "torch_dtype") and config.torch_dtype is not None:
                            torch_dtype = config.torch_dtype
                            logger.info(f"Will use torch_dtype={torch_dtype} as defined in model's config object")
Sylvain Gugger's avatar
Sylvain Gugger committed
3754
                        else:
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
                            if is_sharded and "dtype" in sharded_metadata:
                                torch_dtype = sharded_metadata["dtype"]
                            elif not is_sharded:
                                torch_dtype = get_state_dict_dtype(state_dict)
                            else:
                                one_state_dict = load_state_dict(resolved_archive_file[0])
                                torch_dtype = get_state_dict_dtype(one_state_dict)
                                del one_state_dict  # free CPU memory
                            logger.info(
                                "Since the `torch_dtype` attribute can't be found in model's config object, "
                                "will use torch_dtype={torch_dtype} as derived from model's weights"
                            )
3767
3768
                    elif hasattr(torch, torch_dtype):
                        torch_dtype = getattr(torch, torch_dtype)
3769
3770
                    else:
                        raise ValueError(
3771
                            f'`torch_dtype` can be one of: `torch.dtype`, `"auto"` or a string of a valid `torch.dtype`, but received {torch_dtype}'
3772
3773
3774
                        )
                dtype_orig = cls._set_default_torch_dtype(torch_dtype)

3775
            # Check if `_keep_in_fp32_modules` is not None
3776
            use_keep_in_fp32_modules = (cls._keep_in_fp32_modules is not None) and (
3777
                (torch_dtype == torch.float16) or hasattr(hf_quantizer, "use_keep_in_fp32_modules")
3778
3779
            )

3780
3781
3782
            if is_sharded:
                loaded_state_dict_keys = sharded_metadata["all_checkpoint_keys"]
            else:
3783
                loaded_state_dict_keys = list(state_dict.keys())
3784
3785

            if gguf_path is None and (low_cpu_mem_usage or (use_keep_in_fp32_modules and is_accelerate_available())):
3786
3787
3788
                # In case some weights need to be kept in float32 and accelerate is not installed,
                # we later on want to take the path where state_dict is not None, that is the one
                # that do not require accelerate.
3789
                state_dict = None
3790

3791
3792
        config.name_or_path = pretrained_model_name_or_path

3793
        # Instantiate model.
3794
3795
        init_contexts = [no_init_weights(_enable=_fast_init)]

3796
        if is_deepspeed_zero3_enabled() and not is_quantized:
3797
3798
3799
            import deepspeed

            logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
3800
            init_contexts = [deepspeed.zero.Init(config_dict_or_path=deepspeed_config())] + init_contexts
3801
        elif low_cpu_mem_usage:
3802
3803
            init_contexts.append(init_empty_weights())

3804
3805
3806
3807
        config = copy.deepcopy(config)  # We do not want to modify the config inplace in from_pretrained.
        config = cls._autoset_attn_implementation(
            config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, device_map=device_map
        )
3808

3809
        with ContextManagers(init_contexts):
3810
            # Let's make sure we don't run the init function of buffer modules
3811
3812
            model = cls(config, *model_args, **model_kwargs)

3813
3814
3815
        # If we init with `zero3`, add an attr to the model so we can check downstream for issues
        model._transformers_zero3_init_used = is_deepspeed_zero3_enabled() and not is_quantized

3816
3817
3818
        # make sure we use the model's config since the __init__ call might have copied it
        config = model.config

3819
3820
        # Check first if we are `from_pt`
        if use_keep_in_fp32_modules:
3821
            if is_accelerate_available() and not is_deepspeed_zero3_enabled():
3822
                low_cpu_mem_usage = True
3823
3824
3825
3826
            keep_in_fp32_modules = model._keep_in_fp32_modules
        else:
            keep_in_fp32_modules = []

3827
3828
3829
        if hf_quantizer is not None:
            hf_quantizer.preprocess_model(
                model=model, device_map=device_map, keep_in_fp32_modules=keep_in_fp32_modules
3830
            )
3831

3832
3833
3834
3835
3836
3837
            # We store the original dtype for quantized models as we cannot easily retrieve it
            # once the weights have been quantized
            # Note that once you have loaded a quantized model, you can't change its dtype so this will
            # remain a single source of truth
            config._pre_quantization_dtype = torch_dtype

3838
        if isinstance(device_map, str):
3839
            special_dtypes = {}
3840
3841
3842

            if hf_quantizer is not None:
                special_dtypes.update(hf_quantizer.get_special_dtypes_update(model, torch_dtype))
3843
3844
3845
3846
3847
3848
3849
3850
3851

            special_dtypes.update(
                {
                    name: torch.float32
                    for name, _ in model.named_parameters()
                    if any(m in name for m in keep_in_fp32_modules)
                }
            )

3852
3853
            target_dtype = torch_dtype

3854
3855
            if hf_quantizer is not None:
                target_dtype = hf_quantizer.adjust_target_dtype(target_dtype)
3856

Marc Sun's avatar
Marc Sun committed
3857
            no_split_modules = model._get_no_split_modules(device_map)
3858
3859
3860
3861
3862
            if device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
                raise ValueError(
                    "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or "
                    "'sequential'."
                )
3863

3864
            device_map_kwargs = {"no_split_module_classes": no_split_modules}
3865
            if "special_dtypes" in inspect.signature(infer_auto_device_map).parameters:
3866
                device_map_kwargs["special_dtypes"] = special_dtypes
3867
            elif len(special_dtypes) > 0:
3868
                logger.warning(
3869
3870
3871
                    "This model has some weights that should be kept in higher precision, you need to upgrade "
                    "`accelerate` to properly deal with them (`pip install --upgrade accelerate`)."
                )
3872
            if device_map != "sequential":
3873
3874
                max_memory = get_balanced_memory(
                    model,
3875
                    dtype=target_dtype,
3876
                    low_zero=(device_map == "balanced_low_0"),
3877
                    max_memory=max_memory,
3878
                    **device_map_kwargs,
3879
                )
Marc Sun's avatar
Marc Sun committed
3880
3881
            else:
                max_memory = get_max_memory(max_memory)
3882
3883
            if hf_quantizer is not None:
                max_memory = hf_quantizer.adjust_max_memory(max_memory)
3884
            device_map_kwargs["max_memory"] = max_memory
Marc Sun's avatar
Marc Sun committed
3885

3886
3887
            # Make sure tied weights are tied before creating the device map.
            model.tie_weights()
3888
            device_map = infer_auto_device_map(model, dtype=target_dtype, **device_map_kwargs)
3889

3890
3891
            if hf_quantizer is not None:
                hf_quantizer.validate_environment(device_map=device_map)
3892

3893
3894
3895
3896
        elif device_map is not None:
            model.tie_weights()
            tied_params = find_tied_parameters(model)
            # check if we don't have tied param in different devices
3897
            check_tied_parameters_on_same_device(tied_params, device_map)
3898

3899
        if from_tf:
3900
            if resolved_archive_file.endswith(".index"):
3901
3902
3903
3904
3905
                # Load from a TensorFlow 1.X checkpoint - provided by original authors
                model = cls.load_tf_weights(model, config, resolved_archive_file[:-6])  # Remove the '.index'
            else:
                # Load from our TensorFlow 2.0 checkpoints
                try:
3906
                    from .modeling_tf_pytorch_utils import load_tf2_checkpoint_in_pytorch_model
3907

Yih-Dar's avatar
Yih-Dar committed
3908
3909
3910
                    model, loading_info = load_tf2_checkpoint_in_pytorch_model(
                        model, resolved_archive_file, allow_missing_keys=True, output_loading_info=True
                    )
3911
                except ImportError:
3912
                    logger.error(
Sylvain Gugger's avatar
Sylvain Gugger committed
3913
3914
3915
                        "Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed."
                        " Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation"
                        " instructions."
3916
                    )
3917
                    raise
3918
3919
3920
3921
3922
3923
3924
        elif from_flax:
            try:
                from .modeling_flax_pytorch_utils import load_flax_checkpoint_in_pytorch_model

                model = load_flax_checkpoint_in_pytorch_model(model, resolved_archive_file)
            except ImportError:
                logger.error(
Sylvain Gugger's avatar
Sylvain Gugger committed
3925
3926
3927
                    "Loading a Flax model in PyTorch, requires both PyTorch and Flax to be installed. Please see"
                    " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for"
                    " installation instructions."
3928
3929
                )
                raise
3930
        elif from_pt:
3931
3932
3933
            # restore default dtype
            if dtype_orig is not None:
                torch.set_default_dtype(dtype_orig)
3934

Sylvain Gugger's avatar
Sylvain Gugger committed
3935
3936
3937
3938
3939
3940
3941
3942
            (
                model,
                missing_keys,
                unexpected_keys,
                mismatched_keys,
                offload_index,
                error_msgs,
            ) = cls._load_pretrained_model(
3943
3944
3945
3946
3947
3948
3949
3950
3951
                model,
                state_dict,
                loaded_state_dict_keys,  # XXX: rename?
                resolved_archive_file,
                pretrained_model_name_or_path,
                ignore_mismatched_sizes=ignore_mismatched_sizes,
                sharded_metadata=sharded_metadata,
                _fast_init=_fast_init,
                low_cpu_mem_usage=low_cpu_mem_usage,
3952
3953
3954
3955
                device_map=device_map,
                offload_folder=offload_folder,
                offload_state_dict=offload_state_dict,
                dtype=torch_dtype,
3956
                hf_quantizer=hf_quantizer,
3957
                keep_in_fp32_modules=keep_in_fp32_modules,
3958
                gguf_path=gguf_path,
3959
            )
3960

3961
3962
        # make sure token embedding weights are still tied if needed
        model.tie_weights()
3963

3964
        # Set model in evaluation mode to deactivate DropOut modules by default
3965
3966
        model.eval()

3967
        # If it is a model with generation capabilities, attempt to load the generation config
3968
        if model.can_generate() and pretrained_model_name_or_path is not None:
3969
3970
3971
3972
3973
3974
3975
3976
            try:
                model.generation_config = GenerationConfig.from_pretrained(
                    pretrained_model_name_or_path,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    resume_download=resume_download,
                    proxies=proxies,
                    local_files_only=local_files_only,
3977
                    token=token,
3978
3979
3980
3981
3982
3983
                    revision=revision,
                    subfolder=subfolder,
                    _from_auto=from_auto_class,
                    _from_pipeline=from_pipeline,
                    **kwargs,
                )
3984
            except OSError:
3985
3986
3987
3988
3989
                logger.info(
                    "Generation config file not found, using a generation config created from the model config."
                )
                pass

3990
3991
        # Dispatch model with hooks on all devices if necessary
        if device_map is not None:
3992
3993
3994
3995
            device_map_kwargs = {
                "device_map": device_map,
                "offload_dir": offload_folder,
                "offload_index": offload_index,
3996
                "offload_buffers": offload_buffers,
3997
            }
3998
            if "skip_keys" in inspect.signature(dispatch_model).parameters:
3999
                device_map_kwargs["skip_keys"] = model._skip_keys_device_placement
4000
4001
4002
4003
4004
4005
4006
            # For HQQ method we force-set the hooks for single GPU envs
            if (
                "force_hooks" in inspect.signature(dispatch_model).parameters
                and hf_quantizer is not None
                and hf_quantizer.quantization_config.quant_method == QuantizationMethod.HQQ
            ):
                device_map_kwargs["force_hooks"] = True
Marc Sun's avatar
Marc Sun committed
4007
4008
4009
4010
4011
4012
4013
4014
            if (
                hf_quantizer is not None
                and hf_quantizer.quantization_config.quant_method == QuantizationMethod.FBGEMM_FP8
                and isinstance(device_map, dict)
                and ("cpu" in device_map.values() or "disk" in device_map.values())
            ):
                device_map_kwargs["offload_buffers"] = True

4015
4016
            if not is_fsdp_enabled() and not is_deepspeed_zero3_enabled():
                dispatch_model(model, **device_map_kwargs)
4017

4018
4019
4020
        if hf_quantizer is not None:
            hf_quantizer.postprocess_model(model)
            model.hf_quantizer = hf_quantizer
Marc Sun's avatar
Marc Sun committed
4021

4022
        if _adapter_model_path is not None:
4023
            model.load_adapter(
4024
                _adapter_model_path,
4025
4026
                adapter_name=adapter_name,
                token=token,
4027
                adapter_kwargs=adapter_kwargs,
4028
4029
            )

thomwolf's avatar
thomwolf committed
4030
        if output_loading_info:
Yih-Dar's avatar
Yih-Dar committed
4031
4032
4033
4034
4035
4036
4037
            if loading_info is None:
                loading_info = {
                    "missing_keys": missing_keys,
                    "unexpected_keys": unexpected_keys,
                    "mismatched_keys": mismatched_keys,
                    "error_msgs": error_msgs,
                }
thomwolf's avatar
thomwolf committed
4038
4039
            return model, loading_info

4040
4041
        return model

4042
    @classmethod
Sylvain Gugger's avatar
Sylvain Gugger committed
4043
4044
4045
4046
    def _load_pretrained_model(
        cls,
        model,
        state_dict,
4047
        loaded_keys,
Sylvain Gugger's avatar
Sylvain Gugger committed
4048
4049
4050
4051
4052
        resolved_archive_file,
        pretrained_model_name_or_path,
        ignore_mismatched_sizes=False,
        sharded_metadata=None,
        _fast_init=True,
4053
        low_cpu_mem_usage=False,
4054
4055
        device_map=None,
        offload_folder=None,
4056
        offload_state_dict=None,
4057
        dtype=None,
4058
        hf_quantizer=None,
4059
        keep_in_fp32_modules=None,
4060
        gguf_path=None,
4061
    ):
Sylvain Gugger's avatar
Sylvain Gugger committed
4062
        is_safetensors = False
4063
        is_quantized = hf_quantizer is not None
4064
4065
        state_dict_folder = None
        state_dict_index = None
4066

Sylvain Gugger's avatar
Sylvain Gugger committed
4067
        if device_map is not None and "disk" in device_map.values():
Sylvain Gugger's avatar
Sylvain Gugger committed
4068
4069
4070
4071
4072
            archive_file = (
                resolved_archive_file[0] if isinstance(resolved_archive_file, (list, tuple)) else resolved_archive_file
            )
            is_safetensors = archive_file.endswith(".safetensors")
            if offload_folder is None and not is_safetensors:
Sylvain Gugger's avatar
Sylvain Gugger committed
4073
4074
                raise ValueError(
                    "The current `device_map` had weights offloaded to the disk. Please provide an `offload_folder`"
Sylvain Gugger's avatar
Sylvain Gugger committed
4075
4076
                    " for them. Alternatively, make sure you have `safetensors` installed if the model you are using"
                    " offers the weights in this format."
Sylvain Gugger's avatar
Sylvain Gugger committed
4077
                )
Sylvain Gugger's avatar
Sylvain Gugger committed
4078
4079
            if offload_folder is not None:
                os.makedirs(offload_folder, exist_ok=True)
4080
4081
4082
            if offload_state_dict is None:
                offload_state_dict = True

4083
        is_sharded_safetensors = is_safetensors and sharded_metadata is not None
Patrick von Platen's avatar
Patrick von Platen committed
4084
4085
4086
4087

        # tie the model weights before retrieving the state_dict
        model.tie_weights()

4088
        # Retrieve missing & unexpected_keys
4089
4090
        model_state_dict = model.state_dict()
        expected_keys = list(model_state_dict.keys())
4091
4092
        prefix = model.base_model_prefix

Sylvain Gugger's avatar
Sylvain Gugger committed
4093
4094
4095
4096
4097
4098
4099
        def _fix_key(key):
            if "beta" in key:
                return key.replace("beta", "bias")
            if "gamma" in key:
                return key.replace("gamma", "weight")
            return key

4100
        original_loaded_keys = loaded_keys
Sylvain Gugger's avatar
Sylvain Gugger committed
4101
4102
        loaded_keys = [_fix_key(key) for key in loaded_keys]

4103
4104
4105
4106
4107
4108
        if len(prefix) > 0:
            has_prefix_module = any(s.startswith(prefix) for s in loaded_keys)
            expects_prefix_module = any(s.startswith(prefix) for s in expected_keys)
        else:
            has_prefix_module = False
            expects_prefix_module = False
Patrick von Platen's avatar
Patrick von Platen committed
4109
4110
4111

        # key re-naming operations are never done on the keys
        # that are loaded, but always on the keys of the newly initialized model
4112
4113
        remove_prefix_from_model = not has_prefix_module and expects_prefix_module
        add_prefix_to_model = has_prefix_module and not expects_prefix_module
4114

4115
        if remove_prefix_from_model:
4116
4117
4118
            _prefix = f"{prefix}."
            expected_keys_not_prefixed = [s for s in expected_keys if not s.startswith(_prefix)]
            expected_keys = [s[len(_prefix) :] if s.startswith(_prefix) else s for s in expected_keys]
4119
        elif add_prefix_to_model:
4120
4121
            expected_keys = [".".join([prefix, s]) for s in expected_keys]

4122
        missing_keys = sorted(set(expected_keys) - set(loaded_keys))
Sylvain Gugger's avatar
Sylvain Gugger committed
4123
        unexpected_keys = set(loaded_keys) - set(expected_keys)
4124

Sylvain Gugger's avatar
Sylvain Gugger committed
4125
4126
4127
4128
4129
4130
4131
        # Remove nonpersistent buffers from unexpected keys: they are not in the state dict but will be in the model
        # buffers
        model_buffers = {n for n, _ in model.named_buffers()}
        if remove_prefix_from_model:
            model_buffers = {key[len(_prefix) :] if key.startswith(_prefix) else key for key in model_buffers}
        elif add_prefix_to_model:
            model_buffers = {".".join([prefix, key]) for key in model_buffers}
4132
        unexpected_keys = sorted(unexpected_keys - model_buffers)
4133

4134
        model.tie_weights()
4135
        if device_map is None and not is_fsdp_enabled() and not is_deepspeed_zero3_enabled():
4136
4137
4138
4139
            ptrs = collections.defaultdict(list)
            for name, tensor in model.state_dict().items():
                id_tensor = id_tensor_storage(tensor)
                ptrs[id_tensor].append(name)
Sylvain Gugger's avatar
Sylvain Gugger committed
4140

4141
4142
4143
4144
4145
            # These are all the pointers of shared tensors.
            tied_params = [names for _, names in ptrs.items() if len(names) > 1]
        else:
            # id function doesn't work for meta tensor so we need this function
            tied_params = find_tied_parameters(model)
Sylvain Gugger's avatar
Sylvain Gugger committed
4146
4147

        for group in tied_params:
Sylvain Gugger's avatar
Sylvain Gugger committed
4148
4149
4150
4151
            if remove_prefix_from_model:
                group = [key[len(_prefix) :] if key.startswith(_prefix) else key for key in group]
            elif add_prefix_to_model:
                group = [".".join([prefix, key]) for key in group]
Sylvain Gugger's avatar
Sylvain Gugger committed
4152
4153
4154
            missing_in_group = [k for k in missing_keys if k in group]
            if len(missing_in_group) > 0 and len(missing_in_group) < len(group):
                missing_keys = [k for k in missing_keys if k not in missing_in_group]
4155

4156
4157
4158
4159
4160
4161
4162
4163
4164
        # Some models may have keys that are not in the state by design, removing them before needlessly warning
        # the user.
        if cls._keys_to_ignore_on_load_missing is not None:
            for pat in cls._keys_to_ignore_on_load_missing:
                missing_keys = [k for k in missing_keys if re.search(pat, k) is None]

        if cls._keys_to_ignore_on_load_unexpected is not None:
            for pat in cls._keys_to_ignore_on_load_unexpected:
                unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]
4165
4166
4167
        if hf_quantizer is not None:
            missing_keys = hf_quantizer.update_missing_keys(model, missing_keys, prefix)

4168
4169
4170
4171
        # retrieve weights on meta device and put them back on CPU.
        # This is not ideal in terms of memory, but if we don't do that not, we can't initialize them in the next step
        if low_cpu_mem_usage:
            for key in missing_keys:
Susnato Dhar's avatar
Susnato Dhar committed
4172
4173
                if key in list(model_state_dict.keys()):
                    key = key
4174
4175
                elif f"{prefix}.{key}" in list(model_state_dict.keys()):
                    key = f"{prefix}.{key}"
Susnato Dhar's avatar
Susnato Dhar committed
4176
                elif key.startswith(prefix) and ".".join(key.split(".")[1:]) in list(model_state_dict.keys()):
4177
4178
                    key = ".".join(key.split(".")[1:])
                param = model_state_dict[key]
4179
4180
4181
4182
4183
4184

                # upcast in fp32 if any
                target_dtype = dtype
                if (
                    keep_in_fp32_modules is not None
                    and dtype == torch.float16
4185
4186
4187
                    and any(
                        module_to_keep_in_fp32 in key.split(".") for module_to_keep_in_fp32 in keep_in_fp32_modules
                    )
4188
4189
4190
                ):
                    target_dtype = torch.float32

4191
                if param.device == torch.device("meta"):
4192
                    value = torch.empty(*param.size(), dtype=target_dtype)
4193
                    if (
4194
                        not is_quantized
4195
4196
4197
4198
                        or getattr(hf_quantizer, "requires_parameters_quantization", False)
                        or not hf_quantizer.check_quantized_param(
                            model, param_value=value, param_name=key, state_dict={}
                        )
4199
4200
                    ):
                        set_module_tensor_to_device(model, key, "cpu", value)
4201
                    else:
4202
                        hf_quantizer.create_quantized_param(model, value, key, "cpu", state_dict, unexpected_keys)
4203

4204
        # retrieve uninitialized modules and initialize before maybe overriding that with the pretrained weights.
4205
        if _fast_init:
4206
4207
4208
4209
4210
4211
4212
            if not ignore_mismatched_sizes:
                if remove_prefix_from_model:
                    _loaded_keys = [f"{prefix}.{k}" for k in loaded_keys]
                elif add_prefix_to_model:
                    _loaded_keys = [k[len(prefix) + 1 :] for k in loaded_keys]
                else:
                    _loaded_keys = loaded_keys
4213
                not_initialized_submodules = set_initialized_submodules(model, _loaded_keys)
4214
                # If we're about to tie the output embeds to the input embeds we don't need to init them
4215
4216
4217
                if hasattr(model.config, "tie_word_embeddings") and model.config.tie_word_embeddings:
                    output_embeddings = model.get_output_embeddings()
                    if output_embeddings is not None:
4218
4219
4220
                        # Still need to initialize if there is a bias term since biases are not tied.
                        if not hasattr(output_embeddings, "bias") or output_embeddings.bias is None:
                            output_embeddings._is_hf_initialized = True
4221
4222
            else:
                not_initialized_submodules = dict(model.named_modules())
4223
            # This will only initialize submodules that are not marked as initialized by the line above.
4224
            if is_deepspeed_zero3_enabled() and not is_quantized:
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
                import deepspeed

                not_initialized_parameters = list(
                    set(
                        itertools.chain.from_iterable(
                            submodule.parameters(recurse=False) for submodule in not_initialized_submodules.values()
                        )
                    )
                )
                with deepspeed.zero.GatheredParameters(not_initialized_parameters, modifier_rank=0):
                    model.apply(model._initialize_weights)
            else:
                model.apply(model._initialize_weights)
4238

4239
4240
4241
        # Set some modules to fp32 if any
        if keep_in_fp32_modules is not None:
            for name, param in model.named_parameters():
4242
                if any(module_to_keep_in_fp32 in name.split(".") for module_to_keep_in_fp32 in keep_in_fp32_modules):
4243
4244
                    # param = param.to(torch.float32) does not work here as only in the local scope.
                    param.data = param.data.to(torch.float32)
4245

4246
4247
4248
        # Make sure we are able to load base models as well as derived models (with heads)
        start_prefix = ""
        model_to_load = model
4249
        if len(cls.base_model_prefix) > 0 and not hasattr(model, cls.base_model_prefix) and has_prefix_module:
4250
            start_prefix = cls.base_model_prefix + "."
4251
        if len(cls.base_model_prefix) > 0 and hasattr(model, cls.base_model_prefix) and not has_prefix_module:
4252
            model_to_load = getattr(model, cls.base_model_prefix)
Sylvain Gugger's avatar
Sylvain Gugger committed
4253
4254
            base_model_expected_keys = list(model_to_load.state_dict().keys())
            if any(key in expected_keys_not_prefixed and key not in base_model_expected_keys for key in loaded_keys):
4255
                raise ValueError(
4256
                    "The state dictionary of the model you are trying to load is corrupted. Are you sure it was "
4257
4258
                    "properly saved?"
                )
4259
4260
            if device_map is not None:
                device_map = {k.replace(f"{cls.base_model_prefix}.", ""): v for k, v in device_map.items()}
4261

4262
4263
4264
4265
4266
4267
4268
4269
        def _find_mismatched_keys(
            state_dict,
            model_state_dict,
            loaded_keys,
            add_prefix_to_model,
            remove_prefix_from_model,
            ignore_mismatched_sizes,
        ):
Sylvain Gugger's avatar
Sylvain Gugger committed
4270
4271
4272
            mismatched_keys = []
            if ignore_mismatched_sizes:
                for checkpoint_key in loaded_keys:
4273
4274
4275
                    # If the checkpoint is sharded, we may not have the key here.
                    if checkpoint_key not in state_dict:
                        continue
Sylvain Gugger's avatar
Sylvain Gugger committed
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
                    model_key = checkpoint_key
                    if remove_prefix_from_model:
                        # The model key starts with `prefix` but `checkpoint_key` doesn't so we add it.
                        model_key = f"{prefix}.{checkpoint_key}"
                    elif add_prefix_to_model:
                        # The model key doesn't start with `prefix` but `checkpoint_key` does so we remove it.
                        model_key = ".".join(checkpoint_key.split(".")[1:])

                    if (
                        model_key in model_state_dict
                        and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape
                    ):
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
                        if (
                            state_dict[checkpoint_key].shape[-1] == 1
                            and state_dict[checkpoint_key].numel() * 2 == model_state_dict[model_key].numel()
                        ):
                            # This skips size mismatches for 4-bit weights. Two 4-bit values share an 8-bit container, causing size differences.
                            # Without matching with module type or paramter type it seems like a practical way to detect valid 4bit weights.
                            pass
                        else:
                            mismatched_keys.append(
                                (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape)
                            )
                            del state_dict[checkpoint_key]
4300
4301
            return mismatched_keys

4302
4303
4304
4305
        if resolved_archive_file is not None:
            folder = os.path.sep.join(resolved_archive_file[0].split(os.path.sep)[:-1])
        else:
            folder = None
Sylvain Gugger's avatar
Sylvain Gugger committed
4306
        if device_map is not None and is_safetensors:
4307
            param_device_map = expand_device_map(device_map, original_loaded_keys, start_prefix)
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
            str_dtype = str(dtype).replace("torch.", "") if dtype is not None else "float32"
            if sharded_metadata is None:
                archive_file = (
                    resolved_archive_file[0]
                    if isinstance(resolved_archive_file, (list, tuple))
                    else resolved_archive_file
                )
                weight_map = {p: archive_file for p in original_loaded_keys}
            else:
                weight_map = {p: os.path.join(folder, f) for p, f in sharded_metadata["weight_map"].items()}
Sylvain Gugger's avatar
Sylvain Gugger committed
4318
            offload_index = {
4319
                p[len(start_prefix) :]: {"safetensors_file": f, "weight_name": p, "dtype": str_dtype}
4320
                for p, f in weight_map.items()
4321
                if p.startswith(start_prefix) and param_device_map[p[len(start_prefix) :]] == "disk"
Sylvain Gugger's avatar
Sylvain Gugger committed
4322
            }
4323
4324
        else:
            offload_index = None
Sylvain Gugger's avatar
Sylvain Gugger committed
4325

4326
4327
4328
4329
4330
        if state_dict is not None:
            # Whole checkpoint
            mismatched_keys = _find_mismatched_keys(
                state_dict,
                model_state_dict,
4331
                original_loaded_keys,
4332
4333
4334
4335
                add_prefix_to_model,
                remove_prefix_from_model,
                ignore_mismatched_sizes,
            )
4336

4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
            # For GGUF models `state_dict` is never set to None as the state dict is always small
            if gguf_path:
                error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(
                    model_to_load,
                    state_dict,
                    loaded_keys,
                    start_prefix,
                    expected_keys,
                    device_map=device_map,
                    offload_folder=offload_folder,
                    offload_index=offload_index,
                    state_dict_folder=state_dict_folder,
                    state_dict_index=state_dict_index,
                    dtype=dtype,
                    hf_quantizer=hf_quantizer,
                    is_safetensors=is_safetensors,
                    keep_in_fp32_modules=keep_in_fp32_modules,
                    unexpected_keys=unexpected_keys,
                )
            else:
                # Sharded checkpoint or whole but low_cpu_mem_usage==True
4358
4359
4360
4361
4362
4363
                assign_to_params_buffers = check_support_param_buffer_assignment(
                    model_to_load, state_dict, start_prefix
                )
                error_msgs = _load_state_dict_into_model(
                    model_to_load, state_dict, start_prefix, assign_to_params_buffers
                )
4364
4365

        else:
Sylvain Gugger's avatar
Sylvain Gugger committed
4366
4367
4368
4369
4370
            # This should always be a list but, just to be sure.
            if not isinstance(resolved_archive_file, list):
                resolved_archive_file = [resolved_archive_file]

            error_msgs = []
4371
            mismatched_keys = []
Sylvain Gugger's avatar
Sylvain Gugger committed
4372
4373
            if not is_safetensors:
                offload_index = {} if device_map is not None and "disk" in device_map.values() else None
4374
4375
4376
4377
4378
4379
4380
            if offload_state_dict:
                state_dict_folder = tempfile.mkdtemp()
                state_dict_index = {}
            else:
                state_dict_folder = None
                state_dict_index = None

4381
            if is_sharded_safetensors:
4382
4383
4384
                disk_only_shard_files = get_disk_only_shard_files(
                    device_map, sharded_metadata=sharded_metadata, start_prefix=start_prefix
                )
Sylvain Gugger's avatar
Sylvain Gugger committed
4385
4386
4387
4388
                disk_only_shard_files = [os.path.join(folder, f) for f in disk_only_shard_files]
            else:
                disk_only_shard_files = []

4389
4390
            if len(resolved_archive_file) > 1:
                resolved_archive_file = logging.tqdm(resolved_archive_file, desc="Loading checkpoint shards")
4391
            assign_to_params_buffers = None
Sylvain Gugger's avatar
Sylvain Gugger committed
4392
            for shard_file in resolved_archive_file:
Sylvain Gugger's avatar
Sylvain Gugger committed
4393
4394
4395
                # Skip the load for shards that only contain disk-offloaded weights when using safetensors for the offload.
                if shard_file in disk_only_shard_files:
                    continue
4396
                state_dict = load_state_dict(shard_file, is_quantized=is_quantized)
4397

Sylvain Gugger's avatar
Sylvain Gugger committed
4398
4399
                # Mistmatched keys contains tuples key/shape1/shape2 of weights in the checkpoint that have a shape not
                # matching the weights in the model.
4400
4401
4402
                mismatched_keys += _find_mismatched_keys(
                    state_dict,
                    model_state_dict,
4403
                    original_loaded_keys,
4404
4405
4406
4407
                    add_prefix_to_model,
                    remove_prefix_from_model,
                    ignore_mismatched_sizes,
                )
4408
                if low_cpu_mem_usage:
4409
                    if is_fsdp_enabled() and not is_local_dist_rank_0() and not is_quantized:
4410
4411
                        for key, param in model_to_load.state_dict().items():
                            if param.device == torch.device("meta"):
4412
4413
4414
                                set_module_tensor_to_device(
                                    model_to_load, key, "cpu", torch.empty(*param.size(), dtype=dtype)
                                )
4415
                    else:
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
                        new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(
                            model_to_load,
                            state_dict,
                            loaded_keys,
                            start_prefix,
                            expected_keys,
                            device_map=device_map,
                            offload_folder=offload_folder,
                            offload_index=offload_index,
                            state_dict_folder=state_dict_folder,
                            state_dict_index=state_dict_index,
                            dtype=dtype,
4428
                            hf_quantizer=hf_quantizer,
4429
4430
                            is_safetensors=is_safetensors,
                            keep_in_fp32_modules=keep_in_fp32_modules,
4431
                            unexpected_keys=unexpected_keys,
4432
4433
                        )
                        error_msgs += new_error_msgs
4434
                else:
4435
4436
4437
4438
4439
4440
4441
4442
                    # Sharded checkpoint or whole but low_cpu_mem_usage==True
                    if assign_to_params_buffers is None:
                        assign_to_params_buffers = check_support_param_buffer_assignment(
                            model_to_load, state_dict, start_prefix
                        )
                    error_msgs += _load_state_dict_into_model(
                        model_to_load, state_dict, start_prefix, assign_to_params_buffers
                    )
4443

4444
4445
4446
4447
                # force memory release
                del state_dict
                gc.collect()

4448
            if offload_index is not None and len(offload_index) > 0:
Sylvain Gugger's avatar
Sylvain Gugger committed
4449
4450
4451
                if model != model_to_load:
                    # We need to add the prefix of the base model
                    prefix = cls.base_model_prefix
Sylvain Gugger's avatar
Sylvain Gugger committed
4452
4453
4454
4455
4456
4457
                    if not is_safetensors:
                        for weight_name in offload_index:
                            shutil.move(
                                os.path.join(offload_folder, f"{weight_name}.dat"),
                                os.path.join(offload_folder, f"{prefix}.{weight_name}.dat"),
                            )
Sylvain Gugger's avatar
Sylvain Gugger committed
4458
                    offload_index = {f"{prefix}.{key}": value for key, value in offload_index.items()}
Sylvain Gugger's avatar
Sylvain Gugger committed
4459
4460
4461
                if not is_safetensors:
                    save_offload_index(offload_index, offload_folder)
                    offload_index = None
4462
4463
4464

            if offload_state_dict:
                # Load back temporarily offloaded state dict
4465
                load_offloaded_weights(model_to_load, state_dict_index, state_dict_folder)
4466
4467
                shutil.rmtree(state_dict_folder)

4468
4469
        if len(error_msgs) > 0:
            error_msg = "\n\t".join(error_msgs)
4470
4471
4472
4473
            if "size mismatch" in error_msg:
                error_msg += (
                    "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method."
                )
4474
4475
            raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}")

4476
        if len(unexpected_keys) > 0:
Sylvain Gugger's avatar
Sylvain Gugger committed
4477
            archs = [] if model.config.architectures is None else model.config.architectures
4478
            warner = logger.warning if model.__class__.__name__ in archs else logger.info
Sylvain Gugger's avatar
Sylvain Gugger committed
4479
            warner(
Sylvain Gugger's avatar
Sylvain Gugger committed
4480
4481
4482
4483
4484
4485
4486
                f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"
                f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"
                f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or"
                " with another architecture (e.g. initializing a BertForSequenceClassification model from a"
                " BertForPreTraining model).\n- This IS NOT expected if you are initializing"
                f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly identical"
                " (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)."
4487
4488
4489
4490
4491
            )
        else:
            logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n")
        if len(missing_keys) > 0:
            logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
4492
4493
4494
                f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"
                " TRAIN this model on a down-stream task to be able to use it for predictions and inference."
4495
            )
4496
        elif len(mismatched_keys) == 0:
4497
            logger.info(
Sylvain Gugger's avatar
Sylvain Gugger committed
4498
4499
4500
4501
                f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the checkpoint"
                f" was trained on, you can already use {model.__class__.__name__} for predictions without further"
                " training."
4502
            )
4503
4504
4505
4506
4507
4508
4509
4510
        if len(mismatched_keys) > 0:
            mismatched_warning = "\n".join(
                [
                    f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated"
                    for key, shape1, shape2 in mismatched_keys
                ]
            )
            logger.warning(
Sylvain Gugger's avatar
Sylvain Gugger committed
4511
4512
4513
4514
                f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"
                f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not"
                f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be able"
                " to use it for predictions and inference."
4515
            )
4516

Sylvain Gugger's avatar
Sylvain Gugger committed
4517
        return model, missing_keys, unexpected_keys, mismatched_keys, offload_index, error_msgs
4518
4519

    def retrieve_modules_from_names(self, names, add_prefix=False, remove_prefix=False):
4520
        module_keys = {".".join(key.split(".")[:-1]) for key in names}
4521

Patrick von Platen's avatar
Patrick von Platen committed
4522
4523
        # torch.nn.ParameterList is a special case where two parameter keywords
        # are appended to the module name, *e.g.* bert.special_embeddings.0
4524
        module_keys = module_keys.union(
4525
            {".".join(key.split(".")[:-2]) for key in names if len(key) > 0 and key[-1].isdigit()}
4526
        )
Patrick von Platen's avatar
Patrick von Platen committed
4527

4528
4529
4530
4531
        retrieved_modules = []
        # retrieve all modules that has at least one missing weight name
        for name, module in self.named_modules():
            if remove_prefix:
4532
4533
                _prefix = f"{self.base_model_prefix}."
                name = name[len(_prefix) :] if name.startswith(_prefix) else name
4534
            elif add_prefix:
Patrick von Platen's avatar
Patrick von Platen committed
4535
                name = ".".join([self.base_model_prefix, name]) if len(name) > 0 else self.base_model_prefix
4536
4537
4538
4539
4540
4541

            if name in module_keys:
                retrieved_modules.append(module)

        return retrieved_modules

4542
    @staticmethod
4543
4544
4545
    def _load_pretrained_model_low_mem(
        model, loaded_state_dict_keys, resolved_archive_file, start_prefix="", hf_quantizer=None
    ):
4546
4547
4548
        """
        This is an experimental function that loads the model using ~1.x model size CPU memory

4549
        Before you call it do:
4550

4551
        1. save which state_dict keys are available
4552
4553
4554
4555
4556
4557
4558
4559
        2. drop state_dict before model is created, since the latter takes 1x model size memory

        Here then we continue:

        3. switch to the meta device all params/buffers that are going to be replaced from the loaded state_dict
        4. load state_dict 2nd time
        5. replace the params/buffers from the state_dict

4560
4561
        Currently, it doesn't handle missing_keys, unexpected_keys, mismatched_keys. It can't handle deepspeed. To
        handle bitsandbytes, needs non-empty hf_quantizer argument.
4562
4563
        """

4564
4565
        _move_model_to_meta(model, loaded_state_dict_keys, start_prefix)
        state_dict = load_state_dict(resolved_archive_file)
4566
4567
4568
4569
4570
4571
4572
4573
4574
        expected_keys = loaded_state_dict_keys  # plug for missing expected_keys. TODO: replace with proper keys
        error_msgs = _load_state_dict_into_meta_model(
            model,
            state_dict,
            loaded_state_dict_keys,
            start_prefix,
            expected_keys=expected_keys,
            hf_quantizer=hf_quantizer,
        )
4575
        return error_msgs
4576

4577
4578
4579
4580
4581
4582
    @classmethod
    def register_for_auto_class(cls, auto_class="AutoModel"):
        """
        Register this class with a given auto class. This should only be used for custom models as the ones in the
        library are already mapped with an auto class.

4583
4584
4585
4586
4587
4588
        <Tip warning={true}>

        This API is experimental and may have some slight breaking changes in the next releases.

        </Tip>

4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
        Args:
            auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`):
                The auto class to register this new model with.
        """
        if not isinstance(auto_class, str):
            auto_class = auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(auto_module, auto_class):
            raise ValueError(f"{auto_class} is not a valid auto class.")

        cls._auto_class = auto_class

4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
    def to_bettertransformer(self) -> "PreTrainedModel":
        """
        Converts the model to use [PyTorch's native attention
        implementation](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html), integrated to
        Transformers through [Optimum library](https://huggingface.co/docs/optimum/bettertransformer/overview). Only a
        subset of all Transformers models are supported.

        PyTorch's attention fastpath allows to speed up inference through kernel fusions and the use of [nested
        tensors](https://pytorch.org/docs/stable/nested.html). Detailed benchmarks can be found in [this blog
        post](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2).

        Returns:
            [`PreTrainedModel`]: The model converted to BetterTransformer.
        """
        if not is_optimum_available():
            raise ImportError("The package `optimum` is required to use Better Transformer.")

        from optimum.version import __version__ as optimum_version

        if version.parse(optimum_version) < version.parse("1.7.0"):
            raise ImportError(
                f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
            )

        from optimum.bettertransformer import BetterTransformer

        return BetterTransformer.transform(self)

    def reverse_bettertransformer(self):
        """
        Reverts the transformation from [`~PreTrainedModel.to_bettertransformer`] so that the original modeling is
        used, for example in order to save the model.

        Returns:
            [`PreTrainedModel`]: The model converted back to the original modeling.
        """
        if not is_optimum_available():
            raise ImportError("The package `optimum` is required to use Better Transformer.")

        from optimum.version import __version__ as optimum_version

        if version.parse(optimum_version) < version.parse("1.7.0"):
            raise ImportError(
                f"Please install optimum>=1.7.0 to use Better Transformer. The version {optimum_version} was found."
            )

        from optimum.bettertransformer import BetterTransformer

        return BetterTransformer.reverse(self)

4653
4654
4655
4656
    def warn_if_padding_and_no_attention_mask(self, input_ids, attention_mask):
        """
        Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given.
        """
4657
4658

        # Skip the check during tracing.
4659
        if is_torch_fx_proxy(input_ids) or torch.jit.is_tracing() or is_torchdynamo_compiling():
4660
4661
            return

4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
        if (attention_mask is not None) or (self.config.pad_token_id is None):
            return

        # Check only the first and last input IDs to reduce overhead.
        if self.config.pad_token_id in input_ids[:, [-1, 0]]:
            warn_string = (
                "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See "
                "https://huggingface.co/docs/transformers/troubleshooting"
                "#incorrect-output-when-padding-tokens-arent-masked."
            )

            # If the pad token is equal to either BOS, EOS, or SEP, we do not know whether the user should use an
            # attention_mask or not. In this case, we should still show a warning because this is a rare case.
            if (
                (self.config.bos_token_id is not None and self.config.bos_token_id == self.config.pad_token_id)
                or (self.config.eos_token_id is not None and self.config.eos_token_id == self.config.pad_token_id)
                or (self.config.sep_token_id is not None and self.config.sep_token_id == self.config.pad_token_id)
            ):
                warn_string += (
                    f"\nYou may ignore this warning if your `pad_token_id` ({self.config.pad_token_id}) is identical "
                    f"to the `bos_token_id` ({self.config.bos_token_id}), `eos_token_id` ({self.config.eos_token_id}), "
                    f"or the `sep_token_id` ({self.config.sep_token_id}), and your input is not padded."
                )

            logger.warning_once(warn_string)

4688
4689
    @property
    def _is_quantized_training_enabled(self):
4690
        warnings.warn(
4691
4692
4693
4694
4695
4696
4697
4698
4699
            "`_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead",
            FutureWarning,
        )

        if not hasattr(self, "hf_quantizer"):
            return False

        return self.hf_quantizer.is_trainable

thomwolf's avatar
thomwolf committed
4700

4701
PreTrainedModel.push_to_hub = copy_func(PreTrainedModel.push_to_hub)
4702
4703
4704
4705
if PreTrainedModel.push_to_hub.__doc__ is not None:
    PreTrainedModel.push_to_hub.__doc__ = PreTrainedModel.push_to_hub.__doc__.format(
        object="model", object_class="AutoModel", object_files="model file"
    )
4706
4707


thomwolf's avatar
thomwolf committed
4708
class PoolerStartLogits(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4709
4710
    """
    Compute SQuAD start logits from sequence hidden states.
4711

Sylvain Gugger's avatar
Sylvain Gugger committed
4712
    Args:
4713
4714
        config ([`PretrainedConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
Sylvain Gugger's avatar
Sylvain Gugger committed
4715
4716
4717
    """

    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
4718
        super().__init__()
thomwolf's avatar
thomwolf committed
4719
4720
        self.dense = nn.Linear(config.hidden_size, 1)

Sylvain Gugger's avatar
Sylvain Gugger committed
4721
4722
4723
4724
4725
    def forward(
        self, hidden_states: torch.FloatTensor, p_mask: Optional[torch.FloatTensor] = None
    ) -> torch.FloatTensor:
        """
        Args:
4726
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4727
                The final hidden states of the model.
4728
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4729
4730
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
Sylvain Gugger's avatar
Sylvain Gugger committed
4731
4732

        Returns:
4733
            `torch.FloatTensor`: The start logits for SQuAD.
thomwolf's avatar
thomwolf committed
4734
        """
thomwolf's avatar
thomwolf committed
4735
4736
4737
        x = self.dense(hidden_states).squeeze(-1)

        if p_mask is not None:
Lysandre Debut's avatar
Lysandre Debut committed
4738
            if get_parameter_dtype(self) == torch.float16:
4739
4740
4741
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
4742
4743
4744
4745
4746
4747

        return x


class PoolerEndLogits(nn.Module):
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
4748
    Compute SQuAD end logits from sequence hidden states.
4749

Sylvain Gugger's avatar
Sylvain Gugger committed
4750
    Args:
4751
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4752
4753
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
Sylvain Gugger's avatar
Sylvain Gugger committed
4754
4755
4756
    """

    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
4757
        super().__init__()
thomwolf's avatar
thomwolf committed
4758
4759
4760
4761
4762
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dense_1 = nn.Linear(config.hidden_size, 1)

Sylvain Gugger's avatar
Sylvain Gugger committed
4763
4764
4765
4766
4767
4768
4769
4770
4771
    def forward(
        self,
        hidden_states: torch.FloatTensor,
        start_states: Optional[torch.FloatTensor] = None,
        start_positions: Optional[torch.LongTensor] = None,
        p_mask: Optional[torch.FloatTensor] = None,
    ) -> torch.FloatTensor:
        """
        Args:
4772
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4773
                The final hidden states of the model.
4774
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4775
                The hidden states of the first tokens for the labeled span.
4776
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4777
                The position of the first token for the labeled span.
4778
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4779
4780
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
Sylvain Gugger's avatar
Sylvain Gugger committed
4781

4782
        <Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4783

Stas Bekman's avatar
Stas Bekman committed
4784
4785
        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.
4786
4787

        </Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4788
4789

        Returns:
4790
            `torch.FloatTensor`: The end logits for SQuAD.
thomwolf's avatar
thomwolf committed
4791
        """
4792
4793
4794
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
4795
        if start_positions is not None:
4796
            slen, hsz = hidden_states.shape[-2:]
4797
4798
4799
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions)  # shape (bsz, 1, hsz)
            start_states = start_states.expand(-1, slen, -1)  # shape (bsz, slen, hsz)
thomwolf's avatar
thomwolf committed
4800
4801
4802
4803
4804
4805
4806

        x = self.dense_0(torch.cat([hidden_states, start_states], dim=-1))
        x = self.activation(x)
        x = self.LayerNorm(x)
        x = self.dense_1(x).squeeze(-1)

        if p_mask is not None:
Lysandre Debut's avatar
Lysandre Debut committed
4807
            if get_parameter_dtype(self) == torch.float16:
4808
4809
4810
                x = x * (1 - p_mask) - 65500 * p_mask
            else:
                x = x * (1 - p_mask) - 1e30 * p_mask
thomwolf's avatar
thomwolf committed
4811
4812
4813
4814
4815

        return x


class PoolerAnswerClass(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4816
4817
4818
4819
    """
    Compute SQuAD 2.0 answer class from classification and start tokens hidden states.

    Args:
4820
4821
        config ([`PretrainedConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
Sylvain Gugger's avatar
Sylvain Gugger committed
4822
    """
4823

thomwolf's avatar
thomwolf committed
4824
    def __init__(self, config):
Julien Chaumond's avatar
Julien Chaumond committed
4825
        super().__init__()
thomwolf's avatar
thomwolf committed
4826
4827
4828
4829
        self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
        self.activation = nn.Tanh()
        self.dense_1 = nn.Linear(config.hidden_size, 1, bias=False)

Sylvain Gugger's avatar
Sylvain Gugger committed
4830
4831
4832
4833
4834
4835
4836
    def forward(
        self,
        hidden_states: torch.FloatTensor,
        start_states: Optional[torch.FloatTensor] = None,
        start_positions: Optional[torch.LongTensor] = None,
        cls_index: Optional[torch.LongTensor] = None,
    ) -> torch.FloatTensor:
4837
4838
        """
        Args:
4839
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Sylvain Gugger's avatar
Sylvain Gugger committed
4840
                The final hidden states of the model.
4841
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4842
                The hidden states of the first tokens for the labeled span.
4843
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4844
                The position of the first token for the labeled span.
4845
4846
4847
4848
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.

        <Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4849

Stas Bekman's avatar
Stas Bekman committed
4850
4851
        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.
Sylvain Gugger's avatar
Sylvain Gugger committed
4852

4853
        </Tip>
Sylvain Gugger's avatar
Sylvain Gugger committed
4854
4855

        Returns:
4856
            `torch.FloatTensor`: The SQuAD 2.0 answer class.
thomwolf's avatar
thomwolf committed
4857
        """
Sylvain Gugger's avatar
Sylvain Gugger committed
4858
        # No dependency on end_feature so that we can obtain one single `cls_logits` for each sample.
4859
        hsz = hidden_states.shape[-1]
4860
4861
4862
        assert (
            start_states is not None or start_positions is not None
        ), "One of start_states, start_positions should be not None"
thomwolf's avatar
thomwolf committed
4863
        if start_positions is not None:
4864
4865
            start_positions = start_positions[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            start_states = hidden_states.gather(-2, start_positions).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
4866
4867

        if cls_index is not None:
4868
4869
            cls_index = cls_index[:, None, None].expand(-1, -1, hsz)  # shape (bsz, 1, hsz)
            cls_token_state = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
4870
        else:
4871
            cls_token_state = hidden_states[:, -1, :]  # shape (bsz, hsz)
thomwolf's avatar
thomwolf committed
4872
4873
4874
4875
4876
4877
4878
4879

        x = self.dense_0(torch.cat([start_states, cls_token_state], dim=-1))
        x = self.activation(x)
        x = self.dense_1(x).squeeze(-1)

        return x


4880
4881
4882
@dataclass
class SquadHeadOutput(ModelOutput):
    """
4883
    Base class for outputs of question answering models using a [`~modeling_utils.SQuADHead`].
4884
4885

    Args:
4886
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
Sylvain Gugger's avatar
Sylvain Gugger committed
4887
4888
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification
            losses.
4889
        start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
4890
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
4891
        start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
4892
            Indices for the top config.start_n_top start token possibilities (beam-search).
4893
4894
        end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
Sylvain Gugger's avatar
Sylvain Gugger committed
4895
            (beam-search).
4896
4897
4898
4899
        end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
        cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the `is_impossible` label of the answers.
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910

    """

    loss: Optional[torch.FloatTensor] = None
    start_top_log_probs: Optional[torch.FloatTensor] = None
    start_top_index: Optional[torch.LongTensor] = None
    end_top_log_probs: Optional[torch.FloatTensor] = None
    end_top_index: Optional[torch.LongTensor] = None
    cls_logits: Optional[torch.FloatTensor] = None


thomwolf's avatar
thomwolf committed
4911
class SQuADHead(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
4912
4913
    r"""
    A SQuAD head inspired by XLNet.
4914

Sylvain Gugger's avatar
Sylvain Gugger committed
4915
    Args:
4916
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
4917
4918
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
thomwolf's avatar
thomwolf committed
4919
    """
4920

thomwolf's avatar
thomwolf committed
4921
    def __init__(self, config):
Julien Chaumond's avatar
Julien Chaumond committed
4922
        super().__init__()
thomwolf's avatar
thomwolf committed
4923
4924
4925
4926
4927
4928
4929
        self.start_n_top = config.start_n_top
        self.end_n_top = config.end_n_top

        self.start_logits = PoolerStartLogits(config)
        self.end_logits = PoolerEndLogits(config)
        self.answer_class = PoolerAnswerClass(config)

Sylvain Gugger's avatar
Sylvain Gugger committed
4930
    @replace_return_docstrings(output_type=SquadHeadOutput, config_class=PretrainedConfig)
4931
    def forward(
4932
        self,
Sylvain Gugger's avatar
Sylvain Gugger committed
4933
4934
4935
4936
4937
4938
        hidden_states: torch.FloatTensor,
        start_positions: Optional[torch.LongTensor] = None,
        end_positions: Optional[torch.LongTensor] = None,
        cls_index: Optional[torch.LongTensor] = None,
        is_impossible: Optional[torch.LongTensor] = None,
        p_mask: Optional[torch.FloatTensor] = None,
4939
        return_dict: bool = False,
Sylvain Gugger's avatar
Sylvain Gugger committed
4940
4941
    ) -> Union[SquadHeadOutput, Tuple[torch.FloatTensor]]:
        """
Lysandre's avatar
Lysandre committed
4942
        Args:
4943
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
Lysandre's avatar
Lysandre committed
4944
                Final hidden states of the model on the sequence tokens.
4945
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4946
                Positions of the first token for the labeled span.
4947
            end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4948
                Positions of the last token for the labeled span.
4949
4950
4951
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.
            is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Lysandre's avatar
Lysandre committed
4952
                Whether the question has a possible answer in the paragraph or not.
4953
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
4954
4955
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.
4956
            return_dict (`bool`, *optional*, defaults to `False`):
4957
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
Sylvain Gugger's avatar
Sylvain Gugger committed
4958

Lysandre's avatar
Lysandre committed
4959
        Returns:
Sylvain Gugger's avatar
Sylvain Gugger committed
4960
        """
thomwolf's avatar
thomwolf committed
4961
        start_logits = self.start_logits(hidden_states, p_mask=p_mask)
thomwolf's avatar
thomwolf committed
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984

        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, let's remove the dimension added by batch splitting
            for x in (start_positions, end_positions, cls_index, is_impossible):
                if x is not None and x.dim() > 1:
                    x.squeeze_(-1)

            # during training, compute the end logits based on the ground truth of the start position
            end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask)

            loss_fct = CrossEntropyLoss()
            start_loss = loss_fct(start_logits, start_positions)
            end_loss = loss_fct(end_logits, end_positions)
            total_loss = (start_loss + end_loss) / 2

            if cls_index is not None and is_impossible is not None:
                # Predict answerability from the representation of CLS and START
                cls_logits = self.answer_class(hidden_states, start_positions=start_positions, cls_index=cls_index)
                loss_fct_cls = nn.BCEWithLogitsLoss()
                cls_loss = loss_fct_cls(cls_logits, is_impossible)

                # note(zhiliny): by default multiply the loss by 0.5 so that the scale is comparable to start_loss and end_loss
                total_loss += cls_loss * 0.5
4985

4986
            return SquadHeadOutput(loss=total_loss) if return_dict else (total_loss,)
thomwolf's avatar
thomwolf committed
4987
4988
4989
4990

        else:
            # during inference, compute the end logits based on beam search
            bsz, slen, hsz = hidden_states.size()
4991
            start_log_probs = nn.functional.softmax(start_logits, dim=-1)  # shape (bsz, slen)
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002

            start_top_log_probs, start_top_index = torch.topk(
                start_log_probs, self.start_n_top, dim=-1
            )  # shape (bsz, start_n_top)
            start_top_index_exp = start_top_index.unsqueeze(-1).expand(-1, -1, hsz)  # shape (bsz, start_n_top, hsz)
            start_states = torch.gather(hidden_states, -2, start_top_index_exp)  # shape (bsz, start_n_top, hsz)
            start_states = start_states.unsqueeze(1).expand(-1, slen, -1, -1)  # shape (bsz, slen, start_n_top, hsz)

            hidden_states_expanded = hidden_states.unsqueeze(2).expand_as(
                start_states
            )  # shape (bsz, slen, start_n_top, hsz)
thomwolf's avatar
thomwolf committed
5003
5004
            p_mask = p_mask.unsqueeze(-1) if p_mask is not None else None
            end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask)
5005
            end_log_probs = nn.functional.softmax(end_logits, dim=1)  # shape (bsz, slen, start_n_top)
thomwolf's avatar
thomwolf committed
5006

5007
5008
5009
            end_top_log_probs, end_top_index = torch.topk(
                end_log_probs, self.end_n_top, dim=1
            )  # shape (bsz, end_n_top, start_n_top)
thomwolf's avatar
thomwolf committed
5010
5011
5012
5013
5014
5015
            end_top_log_probs = end_top_log_probs.view(-1, self.start_n_top * self.end_n_top)
            end_top_index = end_top_index.view(-1, self.start_n_top * self.end_n_top)

            start_states = torch.einsum("blh,bl->bh", hidden_states, start_log_probs)
            cls_logits = self.answer_class(hidden_states, start_states=start_states, cls_index=cls_index)

5016
            if not return_dict:
5017
5018
5019
5020
5021
5022
5023
5024
5025
                return (start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits)
            else:
                return SquadHeadOutput(
                    start_top_log_probs=start_top_log_probs,
                    start_top_index=start_top_index,
                    end_top_log_probs=end_top_log_probs,
                    end_top_index=end_top_index,
                    cls_logits=cls_logits,
                )
thomwolf's avatar
thomwolf committed
5026
5027
5028


class SequenceSummary(nn.Module):
Sylvain Gugger's avatar
Sylvain Gugger committed
5029
5030
5031
5032
    r"""
    Compute a single vector summary of a sequence hidden states.

    Args:
5033
        config ([`PretrainedConfig`]):
Sylvain Gugger's avatar
Sylvain Gugger committed
5034
5035
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):
Sylvain Gugger's avatar
Sylvain Gugger committed
5036

5037
            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:
Sylvain Gugger's avatar
Sylvain Gugger committed
5038

5039
5040
5041
5042
5043
                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention
Sylvain Gugger's avatar
Sylvain Gugger committed
5044

5045
            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
Sylvain Gugger's avatar
Sylvain Gugger committed
5046
5047
5048
5049
5050
5051
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
thomwolf's avatar
thomwolf committed
5052
    """
5053

5054
    def __init__(self, config: PretrainedConfig):
Julien Chaumond's avatar
Julien Chaumond committed
5055
        super().__init__()
thomwolf's avatar
thomwolf committed
5056

5057
        self.summary_type = getattr(config, "summary_type", "last")
5058
        if self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
5059
5060
5061
5062
5063
            # We should use a standard multi-head attention module with absolute positional embedding for that.
            # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276
            # We can probably just use the multi-head attention module of PyTorch >=1.1.0
            raise NotImplementedError

thomwolf's avatar
thomwolf committed
5064
        self.summary = Identity()
5065
5066
        if hasattr(config, "summary_use_proj") and config.summary_use_proj:
            if hasattr(config, "summary_proj_to_labels") and config.summary_proj_to_labels and config.num_labels > 0:
5067
                num_classes = config.num_labels
thomwolf's avatar
thomwolf committed
5068
5069
5070
5071
            else:
                num_classes = config.hidden_size
            self.summary = nn.Linear(config.hidden_size, num_classes)

5072
        activation_string = getattr(config, "summary_activation", None)
Lysandre's avatar
Lysandre committed
5073
        self.activation: Callable = get_activation(activation_string) if activation_string else Identity()
thomwolf's avatar
thomwolf committed
5074

thomwolf's avatar
thomwolf committed
5075
        self.first_dropout = Identity()
5076
        if hasattr(config, "summary_first_dropout") and config.summary_first_dropout > 0:
5077
5078
            self.first_dropout = nn.Dropout(config.summary_first_dropout)

thomwolf's avatar
thomwolf committed
5079
        self.last_dropout = Identity()
5080
        if hasattr(config, "summary_last_dropout") and config.summary_last_dropout > 0:
5081
            self.last_dropout = nn.Dropout(config.summary_last_dropout)
thomwolf's avatar
thomwolf committed
5082

Sylvain Gugger's avatar
Sylvain Gugger committed
5083
5084
5085
5086
5087
5088
5089
    def forward(
        self, hidden_states: torch.FloatTensor, cls_index: Optional[torch.LongTensor] = None
    ) -> torch.FloatTensor:
        """
        Compute a single vector summary of a sequence hidden states.

        Args:
5090
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
Sylvain Gugger's avatar
Sylvain Gugger committed
5091
                The hidden states of the last layer.
5092
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
Sylvain Gugger's avatar
Sylvain Gugger committed
5093
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.
Sylvain Gugger's avatar
Sylvain Gugger committed
5094
5095

        Returns:
5096
            `torch.FloatTensor`: The summary of the sequence hidden states.
thomwolf's avatar
thomwolf committed
5097
        """
5098
        if self.summary_type == "last":
thomwolf's avatar
thomwolf committed
5099
            output = hidden_states[:, -1]
5100
        elif self.summary_type == "first":
thomwolf's avatar
thomwolf committed
5101
            output = hidden_states[:, 0]
5102
        elif self.summary_type == "mean":
thomwolf's avatar
thomwolf committed
5103
            output = hidden_states.mean(dim=1)
5104
        elif self.summary_type == "cls_index":
thomwolf's avatar
thomwolf committed
5105
            if cls_index is None:
Lysandre's avatar
Lysandre committed
5106
5107
5108
5109
5110
                cls_index = torch.full_like(
                    hidden_states[..., :1, :],
                    hidden_states.shape[-2] - 1,
                    dtype=torch.long,
                )
thomwolf's avatar
thomwolf committed
5111
            else:
thomwolf's avatar
thomwolf committed
5112
                cls_index = cls_index.unsqueeze(-1).unsqueeze(-1)
5113
                cls_index = cls_index.expand((-1,) * (cls_index.dim() - 1) + (hidden_states.size(-1),))
thomwolf's avatar
thomwolf committed
5114
            # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states
5115
5116
            output = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, XX, hidden_size)
        elif self.summary_type == "attn":
thomwolf's avatar
thomwolf committed
5117
5118
            raise NotImplementedError

5119
        output = self.first_dropout(output)
thomwolf's avatar
thomwolf committed
5120
5121
        output = self.summary(output)
        output = self.activation(output)
5122
        output = self.last_dropout(output)
thomwolf's avatar
thomwolf committed
5123
5124
5125
5126

        return output


5127
def unwrap_model(model: nn.Module, recursive: bool = False) -> nn.Module:
5128
5129
5130
5131
    """
    Recursively unwraps a model from potential containers (as used in distributed training).

    Args:
5132
        model (`torch.nn.Module`): The model to unwrap.
5133
5134
5135
        recursive (`bool`, *optional*, defaults to `False`):
            Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers
            recursively, not just the top-level distributed containers.
5136
    """
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
    # Use accelerate implementation if available (should always be the case when using torch)
    # This is for pytorch, as we also have to handle things like dynamo
    if is_accelerate_available():
        kwargs = {}
        if recursive:
            if not is_accelerate_available("0.29.0"):
                raise RuntimeError(
                    "Setting `recursive=True` to `unwrap_model` requires `accelerate` v0.29.0. Please upgrade your version of accelerate"
                )
            else:
                kwargs["recursive"] = recursive
        return extract_model_from_parallel(model, **kwargs)
5149
    else:
5150
5151
5152
5153
5154
        # since there could be multiple levels of wrapping, unwrap recursively
        if hasattr(model, "module"):
            return unwrap_model(model.module)
        else:
            return model
Sylvain Gugger's avatar
Sylvain Gugger committed
5155
5156


5157
def expand_device_map(device_map, param_names, start_prefix):
Sylvain Gugger's avatar
Sylvain Gugger committed
5158
5159
5160
5161
    """
    Expand a device map to return the correspondance parameter name to device.
    """
    new_device_map = {}
5162
    param_names = [p[len(start_prefix) :] for p in param_names if p.startswith(start_prefix)]
Sylvain Gugger's avatar
Sylvain Gugger committed
5163
    for module, device in device_map.items():
5164
5165
5166
        new_device_map.update(
            {p: device for p in param_names if p == module or p.startswith(f"{module}.") or module == ""}
        )
Sylvain Gugger's avatar
Sylvain Gugger committed
5167
5168
5169
    return new_device_map


5170
def get_disk_only_shard_files(device_map, sharded_metadata, start_prefix):
Sylvain Gugger's avatar
Sylvain Gugger committed
5171
5172
5173
    """
    Returns the list of shard files containing only weights offloaded to disk.
    """
5174
5175
5176
5177

    weight_map = {
        p[len(start_prefix) :]: v for p, v in sharded_metadata["weight_map"].items() if p.startswith(start_prefix)
    }
Sylvain Gugger's avatar
Sylvain Gugger committed
5178
    files_content = collections.defaultdict(list)
5179
    for weight_name, filename in weight_map.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
5180
5181
5182
5183
5184
        while len(weight_name) > 0 and weight_name not in device_map:
            weight_name = ".".join(weight_name.split(".")[:-1])
        files_content[filename].append(device_map[weight_name])

    return [fname for fname, devices in files_content.items() if set(devices) == {"disk"}]