config.py 38.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
5
from collections.abc import Callable
6
from dataclasses import asdict
7
from functools import cache, partial
8
from importlib.metadata import version
9
from pathlib import Path
10
from typing import Any, Literal, TypeAlias
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
14
15
from huggingface_hub import (
    get_safetensors_metadata,
)
16
from packaging.version import Version
17
from transformers import GenerationConfig, PretrainedConfig
18
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
19
from transformers.models.auto.image_processing_auto import get_image_processor_config
20
21
22
23
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
24
from transformers.models.auto.tokenization_auto import get_tokenizer_config
25
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26

27
from vllm import envs
28
from vllm.logger import init_logger
29
30
31

from .config_parser_base import ConfigParserBase
from .repo_utils import (
32
33
34
35
36
37
38
    _get_hf_token,
    file_or_path_exists,
    get_hf_file_to_dict,
    list_repo_files,
    try_get_local_file,
    with_retry,
)
39
from .utils import (
40
    check_gguf_file,
41
42
    is_gguf,
    is_remote_gguf,
43
    parse_safetensors_file_metadata,
44
    split_remote_gguf,
45
)
46

47
if envs.VLLM_USE_MODELSCOPE:
48
49
50
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
51

52
53
MISTRAL_CONFIG_NAME = "params.json"

54
55
logger = init_logger(__name__)

56

57
58
class LazyConfigDict(dict):
    def __getitem__(self, key):
59
60
61
        if isinstance(value := super().__getitem__(key), type):
            return value

62
        import vllm.transformers_utils.configs as configs
63

64
        return getattr(configs, value)
65
66
67


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
68
    afmoe="AfmoeConfig",
69
70
    chatglm="ChatGLMConfig",
    deepseek_vl_v2="DeepseekVLV2Config",
71
    deepseek_v32="DeepseekV3Config",
72
    flex_olmo="FlexOlmoConfig",
73
    hunyuan_vl="HunYuanVLConfig",
74
    kimi_linear="KimiLinearConfig",
75
76
77
78
79
80
    kimi_vl="KimiVLConfig",
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
81
    midashenglm="MiDashengLMConfig",
82
83
84
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
85
    olmo3="Olmo3Config",
86
87
88
89
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
90
    qwen3_next="Qwen3NextConfig",
Paul Pak's avatar
Paul Pak committed
91
    lfm2_moe="Lfm2MoeConfig",
92
)
93

94
95
96
97
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

98
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
99
    "internvl_chat": {"has_no_defaults_at_init": True},
100
    "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"},
101
    "NVLM_D": {"has_no_defaults_at_init": True},
102
103
}

104

105
class HFConfigParser(ConfigParserBase):
106
107
    def parse(
        self,
108
        model: str | Path,
109
        trust_remote_code: bool,
110
111
        revision: str | None = None,
        code_revision: str | None = None,
112
113
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
114
115
116
117
118
119
120
121
122
123
124
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            token=_get_hf_token(),
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
125
126
127
128
129
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
130
131
132
133
134
135
136
137
138
139
140
141

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        else:
            try:
142
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
143
144
145
146
147
148
149
150
151
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    token=_get_hf_token(),
                    **kwargs,
                )
            except ValueError as e:
152
153
154
155
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
156
157
158
159
160
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
161
162
                        "`--trust-remote-code` flag in the CLI."
                    )
163
164
165
166
167
168
169
170
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
171
172
    def parse(
        self,
173
        model: str | Path,
174
        trust_remote_code: bool,
175
176
        revision: str | None = None,
        code_revision: str | None = None,
177
178
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
179
180
181
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
182
183
184
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
185
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
186
187
                model, revision, **kwargs
            )
188
189
190
191
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

192
193
194
195
196
197
198
199
200
201
202
203
204
        # Get missing fields from HF config if available
        try:
            hf_config_dict, _ = PretrainedConfig.get_config_dict(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        except OSError:  # Not found
            hf_config_dict = {}

        config = adapt_config_dict(config_dict, defaults=hf_config_dict)
205
206
207

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
208
209
210
        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
            sliding_window, list
        ):
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
259
260
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
261
262
263
264
265
266
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
267
268
269
270
271
272
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
273
274
275
276
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
277
        if not issubclass(config_parser_cls, ConfigParserBase):
278
279
280
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
281
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
282
283
284
285
286
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
287
288
289
        return config_parser_cls

    return _wrapper
290
291


292
293
294
295
296
297
298
299
300
301
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
    """Some models may have no rope_theta in their config but still use RoPE.
    This function sets a default rope_theta if it's missing."""
    if getattr(config, "rope_parameters", None) is None:
        config.rope_parameters = {"rope_type": "default"}
    if "rope_theta" not in config.rope_parameters:
        config.rope_parameters["rope_theta"] = default_theta


def patch_rope_parameters(config: PretrainedConfig) -> None:
302
    """Provide backwards compatibility for RoPE."""
303
    # Patch rope_parameters differently based on Transformers version
304
    if Version(version("transformers")) >= Version("5.0.0.dev0"):
305
306
307
        from transformers.modeling_rope_utils import (
            rope_config_validation,
            standardize_rope_params,
308
        )
309
310
311
312
313
314
315
316

        # When Transformers v5 is installed, legacy rope_theta may be present
        # when using custom code models written for Transformers v4
        if (rope_theta := getattr(config, "rope_theta", None)) is not None:
            standardize_rope_params(config, rope_theta=rope_theta)
            rope_config_validation(config)
            # Delete rope_theta to avoid confusion in downstream code
            del config.rope_theta
317
    else:
318
319
320
321
322
323
324
325
        # When Transformers v4 is installed, legacy rope_scaling may be present
        if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
            config.rope_parameters = rope_scaling
        # When Transformers v4 is installed, legacy rope_theta may be present
        if (rope_theta := getattr(config, "rope_theta", None)) is not None:
            if not hasattr(config, "rope_parameters"):
                config.rope_parameters = {"rope_type": "default"}
            config.rope_parameters["rope_theta"] = rope_theta
326
327

    # No RoPE parameters to patch
328
    if not hasattr(config, "rope_parameters"):
329
330
        return

331
332
333
334
    # Add original_max_position_embeddings if present
    if ompe := getattr(config, "original_max_position_embeddings", None):
        config.rope_parameters["original_max_position_embeddings"] = ompe

335
    # Handle nested rope_parameters in interleaved sliding attention models
336
337
    if set(config.rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
        for rope_parameters_layer_type in config.rope_parameters.values():
338
339
            patch_rope_parameters_dict(rope_parameters_layer_type)
    else:
340
        patch_rope_parameters_dict(config.rope_parameters)
341
342


343
344
345
346
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
    if "rope_type" in rope_parameters and "type" in rope_parameters:
        rope_type = rope_parameters["rope_type"]
        rope_type_legacy = rope_parameters["type"]
347
348
349
350
351
        if (rope_type_legacy == "su" and rope_type == "longrope") or (
            rope_type_legacy == "mrope" and rope_type == "default"
        ):
            pass  # No action needed
        elif rope_type != rope_type_legacy:
352
353
354
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
355
356
                "You should only specify one of them."
            )
357

358
359
    if "rope_type" not in rope_parameters and "type" in rope_parameters:
        rope_parameters["rope_type"] = rope_parameters["type"]
360
361
        logger.info("Replacing legacy 'type' key with 'rope_type'")

362
363
    if "rope_type" not in rope_parameters:
        raise ValueError("rope_parameters should have a 'rope_type' key")
364

365
366
    if rope_parameters["rope_type"] == "su":
        rope_parameters["rope_type"] = "longrope"
367
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
368
369
370
    elif rope_parameters["rope_type"] == "mrope":
        assert "mrope_section" in rope_parameters
        rope_parameters["rope_type"] = "default"
371
372
373
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


374
def _uses_mrope(config: PretrainedConfig) -> bool:
375
376
    rope_parameters = getattr(config, "rope_parameters", None)
    if rope_parameters is None:
377
378
        return False

379
    return "mrope_section" in rope_parameters
380
381


382
383
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
384
385
386
387
388
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
def uses_xdrope_dim(config: PretrainedConfig) -> int:
    """Detect if the model with this config uses XD-ROPE."""
    xdrope_section = getattr(config, "xdrope_section", None)
    if xdrope_section is not None and isinstance(xdrope_section, list):
        return len(xdrope_section)
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return 0

    if isinstance(rope_scaling, dict) and "xdrope_section" in rope_scaling:
        xdrope_section = rope_scaling["xdrope_section"]
        if xdrope_section is not None and isinstance(xdrope_section, list):
            return len(xdrope_section)

    return 0


421
422
423
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

424
425
426
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

427
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
428
429


430
431
432
433
434
435
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
436
        return len(set(layer_types)) > 1
437
438
439
    return False


440
441
442
443
444
445
446
447
448
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


449
450
451
452
453
454
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
455
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
456
457
458
    return config


459
def maybe_override_with_speculators(
460
    model: str,
461
    tokenizer: str | None,
462
    trust_remote_code: bool,
463
464
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
465
    **kwargs,
466
) -> tuple[str, str | None, dict[str, Any] | None]:
467
    """
468
469
470
471
472
473
474
475
476
477
478
479
480
481
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
482
    """
483
    if check_gguf_file(model):
484
485
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
486
487
488
    elif is_remote_gguf(model):
        repo_id, _ = split_remote_gguf(model)
        gguf_model_repo = Path(repo_id)
489
490
    else:
        gguf_model_repo = None
491
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
492
    config_dict, _ = PretrainedConfig.get_config_dict(
493
        model if gguf_model_repo is None else gguf_model_repo,
494
495
496
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
497
        **kwargs,
498
    )
499
500
501
502
503
504
505
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
506
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
507

508
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
509
510
        config_dict=config_dict
    )
511
512

    # Set the draft model to the speculators model
513
    speculative_config["model"] = model
514
515
516
517
518

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

519
    return model, tokenizer, speculative_config
520
521


522
def get_config(
523
    model: str | Path,
524
    trust_remote_code: bool,
525
526
527
528
529
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
530
531
532
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
533

534
535
536
537
538
539
540
541
542
543
544
545
    _is_gguf = is_gguf(model)
    _is_remote_gguf = is_remote_gguf(model)
    if _is_gguf:
        if check_gguf_file(model):
            # Local GGUF file
            kwargs["gguf_file"] = Path(model).name
            model = Path(model).parent
        elif _is_remote_gguf:
            # Remote GGUF - extract repo_id from repo_id:quant_type format
            # The actual GGUF file will be downloaded later by GGUFModelLoader
            # Keep model as repo_id:quant_type for download, but use repo_id for config
            model, _ = split_remote_gguf(model)
546

547
    if config_format == "auto":
548
        try:
549
550
551
            # First check for Mistral to avoid defaulting to
            # Transformers implementation.
            if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
552
                config_format = "mistral"
553
            elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
554
555
556
                model, HF_CONFIG_NAME, revision=revision
            ):
                config_format = "hf"
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
            # Remote GGUF models must have config.json in repo,
            # otherwise the config can't be parsed correctly.
            # FIXME(Isotr0py): Support remote GGUF repos without config.json
            elif _is_remote_gguf and not file_or_path_exists(
                model, HF_CONFIG_NAME, revision=revision
            ):
                err_msg = (
                    "Could not find config.json for remote GGUF model repo. "
                    "To load remote GGUF model through `<repo_id>:<quant_type>`, "
                    "ensure your model has config.json (HF format) file. "
                    "Otherwise please specify --hf-config-path <original_repo> "
                    "in engine args to fetch config from unquantized hf model."
                )
                logger.error(err_msg)
                raise ValueError(err_msg)
572
573
574
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
575
576
577
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
578
579
                    "in engine args for customized config parser."
                )
580
581
582
583
584
585
586
587
588
589
590

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
591
                "'params.json'.\n"
592
            ).format(model=model)
593
594

            raise ValueError(error_message) from e
595

596
597
598
599
600
601
602
603
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
        **kwargs,
    )
604
    # Special architecture mapping check for GGUF models
605
    if _is_gguf:
606
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
607
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
608
609
610
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

611
612
613
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
614
615
616
617
618
619
620
621
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
622

623
624
625
626
627
628
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
629
630
631
632
633
634
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
635
636
637

    if quantization_config is not None:
        config.quantization_config = quantization_config
638
        # auto-enable DeepGEMM UE8M0 if model config requests it
639
        scale_fmt = quantization_config.get("scale_fmt", None)
640
        if scale_fmt in ("ue8m0",):
641
642
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
643
                logger.info_once(
644
645
                    (
                        "Detected quantization_config.scale_fmt=%s; "
646
                        "enabling UE8M0 for DeepGEMM."
647
                    ),
648
649
                    scale_fmt,
                )
650
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
651
                logger.warning_once(
652
653
654
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
655
656
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
657
                    ),
658
659
                    scale_fmt,
                )
660

661
662
663
664
665
666
667
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

668
669
670
671
672
673
674
675
    # Exhaustively patch RoPE parameters everywhere they might be
    patch_rope_parameters(config)
    patch_rope_parameters(config.get_text_config())
    SubConfigs: TypeAlias = dict[str, PretrainedConfig]
    sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
    if sub_configs:
        for sub_config in sub_configs:
            patch_rope_parameters(getattr(config, sub_config))
676

677
678
679
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

680
    return config
681
682


683
@cache
684
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None:
685
    """
686
687
688
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
689
690

    Args:
691
        model: The name of the Hugging Face model.
692
        revision: The specific version of the model to use.
693
            Defaults to 'main'.
694
695

    Returns:
696
        A dictionary containing the pooling type and whether
697
            normalization is used, or None if no pooling configuration is found.
698
    """
699
700
    if is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
701
702

    modules_file_name = "modules.json"
703
704

    modules_dict = None
705
706
707
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
708
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
709
710
711
712

    if modules_dict is None:
        return None

713
714
    logger.info("Found sentence-transformers modules configuration.")

715
716
717
718
719
720
721
722
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
723
    normalize = bool(
724
725
726
727
728
729
730
731
732
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
733
734
735

    if pooling:
        pooling_file_name = "{}/config.json".format(pooling["path"])
736
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
737
        pooling_type_name = next(
738
739
            (item for item, val in pooling_dict.items() if val is True), None
        )
740
741
742
743

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

744
        logger.info("Found pooling configuration.")
745
746
747
748
749
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


750
def get_pooling_config_name(pooling_name: str) -> str | None:
751
752
753
754
755
756
757
758
759
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

760
    supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"]
761
762
    pooling_type_name = pooling_name.upper()

763
764
765
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

766
    raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
767
768


769
@cache
770
def get_sentence_transformer_tokenizer_config(
771
    model: str | Path, revision: str | None = "main"
772
):
773
    """
774
    Returns the tokenization configuration dictionary for a
775
776
777
    given Sentence Transformer BERT model.

    Parameters:
778
    - model (str|Path): The name of the Sentence Transformer
779
780
781
782
783
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
784
    - dict: A dictionary containing the configuration parameters
785
786
    for the Sentence Transformer BERT model.
    """
787
788
789
790
791
792
793
794
795
796
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
797
798

    for config_file in sentence_transformer_config_files:
799
800
801
802
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
803
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
804
805
            if encoder_dict:
                break
806

807
    if not encoder_dict and not Path(model).is_absolute():
808
809
        try:
            # If model is on HuggingfaceHub, get the repo files
810
811
812
            repo_files = list_repo_files(
                model, revision=revision, token=_get_hf_token()
            )
813
        except Exception:
814
815
816
817
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
818
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
819
820
821
                if encoder_dict:
                    break

822
823
824
    if not encoder_dict:
        return None

825
826
    logger.info("Found sentence-transformers tokenize configuration.")

827
828
829
830
831
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


832
def maybe_register_config_serialize_by_value() -> None:
833
834
    """Try to register HF model configuration class to serialize by value

835
836
837
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
838

839
    Examples:
840

841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
865
866
    try:
        import transformers_modules
867

868
        transformers_modules_available = True
869
    except ImportError:
870
        transformers_modules_available = False
871
872
873
874
875

    try:
        import multiprocessing
        import pickle

876
877
        import cloudpickle

878
        from vllm.config import VllmConfig
879

880
881
882
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
883
        def _reduce_config(config: VllmConfig):
884
            return (pickle.loads, (cloudpickle.dumps(config),))
885

886
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
887

888
889
890
891
892
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
893
            from vllm.v1.executor.ray_utils import ray
894

895
896
897
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

898
899
900
901
902
903
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
904
905
            exc_info=e,
        )
906
907


908
def get_hf_image_processor_config(
909
910
911
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
912
    **kwargs,
913
) -> dict[str, Any]:
914
    # ModelScope does not provide an interface for image_processor
915
    if envs.VLLM_USE_MODELSCOPE:
916
        return dict()
917
    # Separate model folder from file path for GGUF models
918
    if check_gguf_file(model):
919
        model = Path(model).parent
920
921
    elif is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
922
923
924
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
925
926


927
928
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
929
    No op for pure text models.
930
    """
931
932
933
934
935
936
937
938
939
    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
940
941
942
943
944


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
945
946
947
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
) -> GenerationConfig | None:
948
949
950
951
952
953
954
955
956
957
958
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
959
                config_format=config_format,
960
961
962
963
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
964
965


966
967
968
def try_get_safetensors_metadata(
    model: str,
    *,
969
    revision: str | None = None,
970
971
972
973
974
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
975
        token=_get_hf_token(),
976
977
978
    )

    try:
979
980
981
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
982
983
    except Exception:
        return None
984
985
986


def try_get_tokenizer_config(
987
    pretrained_model_name_or_path: str | os.PathLike,
988
    trust_remote_code: bool,
989
990
    revision: str | None = None,
) -> dict[str, Any] | None:
991
992
993
994
995
996
997
998
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
999
1000


1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

        dense_modules = [
            m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
        ]
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1035
1036
1037
def get_safetensors_params_metadata(
    model: str,
    *,
1038
    revision: str | None = None,
1039
1040
1041
1042
1043
1044
1045
1046
1047
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1048
1049
1050
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1063
1064
1065
1066
1067
1068
1069
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1070
1071
            f"and if the config file exists."
        )
1072
1073
1074
1075
1076
1077
1078
1079
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1080
1081
1082
1083
1084
1085
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1086
1087
1088
1089
1090
1091
1092
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1093
1094
            exc_info=e,
        )
1095
1096

    return max_position_embeddings