config.py 45.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
Julien Denize's avatar
Julien Denize committed
5
6
from collections.abc import Callable, Iterator
from contextlib import contextmanager
7
from dataclasses import asdict
8
from functools import cache, partial
9
from importlib.metadata import version
10
from pathlib import Path
11
from typing import Any, Literal, TypeAlias
Jasmond L's avatar
Jasmond L committed
12

Joe Runde's avatar
Joe Runde committed
13
import huggingface_hub
Julien Denize's avatar
Julien Denize committed
14
15
import torch
from huggingface_hub import constants, get_safetensors_metadata
16
from packaging.version import Version
Julien Denize's avatar
Julien Denize committed
17
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
18
from transformers import GenerationConfig, PretrainedConfig
19
from transformers.models.auto.image_processing_auto import get_image_processor_config
20
21
22
23
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
24
from transformers.models.auto.tokenization_auto import get_tokenizer_config
25
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26

27
from vllm import envs
28
from vllm.logger import init_logger
29
from vllm.transformers_utils.repo_utils import is_mistral_model_repo
30
31
32
33
from vllm.transformers_utils.utils import (
    parse_safetensors_file_metadata,
    without_trust_remote_code,
)
Julien Denize's avatar
Julien Denize committed
34
from vllm.utils.torch_utils import common_broadcastable_dtype
35
36

from .config_parser_base import ConfigParserBase
37
38
39
40
41
42
from .gguf_utils import (
    check_gguf_file,
    is_gguf,
    is_remote_gguf,
    split_remote_gguf,
)
43
from .repo_utils import (
44
45
46
47
48
49
    file_or_path_exists,
    get_hf_file_to_dict,
    list_repo_files,
    try_get_local_file,
    with_retry,
)
50

51
52
53
54
55
56
57
58
59
try:
    # Transformers v5
    from transformers.configuration_utils import ALLOWED_ATTENTION_LAYER_TYPES
except ImportError:
    # Transformers v4
    from transformers.configuration_utils import (
        ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
    )

60
if envs.VLLM_USE_MODELSCOPE:
61
62
63
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
64

65
66
MISTRAL_CONFIG_NAME = "params.json"

67
68
logger = init_logger(__name__)

69

70
71
class LazyConfigDict(dict):
    def __getitem__(self, key):
72
73
74
        if isinstance(value := super().__getitem__(key), type):
            return value

75
        import vllm.transformers_utils.configs as configs
76

77
        return getattr(configs, value)
78
79
80


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
81
    afmoe="AfmoeConfig",
82
    bagel="BagelConfig",
83
    umm="CheersConfig",
84
    chatglm="ChatGLMConfig",
85
    modernvbert="ColModernVBertConfig",
86
    colpali="ColPaliConfig",
87
88
    colqwen3="ColQwen3Config",
    ops_colqwen3="OpsColQwen3Config",
89
    qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
90
    deepseek_vl_v2="DeepseekVLV2Config",
91
    deepseek_v32="DeepseekV3Config",
92
    deepseek_v4="DeepseekV4Config",
93
    flex_olmo="FlexOlmoConfig",
94
    fireredlid="FireRedLIDConfig",
95
    funaudiochat="FunAudioChatConfig",
96
    granite4_vision="Granite4VisionConfig",
97
    hunyuan_vl="HunYuanVLConfig",
98
    hy_v3="HYV3Config",
oscardev256's avatar
oscardev256 committed
99
    isaac="IsaacConfig",
100
    kimi_k2="DeepseekV3Config",  # Kimi K2 uses same architecture as DeepSeek V3
101
    kimi_linear="KimiLinearConfig",
102
    kimi_vl="KimiVLConfig",
Roger Wang's avatar
Roger Wang committed
103
    kimi_k25="KimiK25Config",
104
105
106
107
108
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
109
    midashenglm="MiDashengLMConfig",
110
111
112
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
113
    olmo_hybrid="OlmoHybridConfig",
114
115
116
117
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
csy0225's avatar
csy0225 committed
118
    step3p5="Step3p5Config",
Roger Wang's avatar
Roger Wang committed
119
    qwen3_asr="Qwen3ASRConfig",
120
    qwen3_next="Qwen3NextConfig",
121
122
    qwen3_5="Qwen3_5Config",
    qwen3_5_moe="Qwen3_5MoeConfig",
Paul Pak's avatar
Paul Pak committed
123
    lfm2_moe="Lfm2MoeConfig",
124
    tarsier2="Tarsier2Config",
125
)
126

127
128
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"}

129
130
131
132
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

133
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
134
    "internvl_chat": {"has_no_defaults_at_init": True},
135
    "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"},
136
    "NVLM_D": {"has_no_defaults_at_init": True},
137
138
}

139

140
141
142
143
144
145
146
147
def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool:
    """Check if rope_parameters is nested by layer types."""
    # Cannot be nested if rope_parameters is empty
    if not rope_parameters:
        return False
    return set(rope_parameters.keys()).issubset(ALLOWED_ATTENTION_LAYER_TYPES)


Julien Denize's avatar
Julien Denize committed
148
149
150
151
152
153
154
155
156
157
158
159
160
@contextmanager
def _mistral_patch_hf_hub_constants() -> Iterator[None]:
    hf_safetensors_single_file = constants.SAFETENSORS_SINGLE_FILE
    hf_safetensors_index_file = constants.SAFETENSORS_INDEX_FILE
    constants.SAFETENSORS_SINGLE_FILE = "consolidated.safetensors"
    constants.SAFETENSORS_INDEX_FILE = "consolidated.safetensors.index.json"
    try:
        yield
    finally:
        constants.SAFETENSORS_SINGLE_FILE = hf_safetensors_single_file
        constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file


161
class HFConfigParser(ConfigParserBase):
162
163
    def parse(
        self,
164
        model: str | Path,
165
        trust_remote_code: bool,
166
167
        revision: str | None = None,
        code_revision: str | None = None,
168
169
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
170
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
171
172
        trust_remote_code |= kwargs.get("trust_remote_code", False)
        kwargs = without_trust_remote_code(kwargs)
173
174
175
176
177
178
179
180
181
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
182
183
184
185
186
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
187
188
        # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
        if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
189
190
191
192
193
194
195
196
197
198
            if isinstance(hf_overrides, dict) and "model_type" in hf_overrides:
                model_type = hf_overrides["model_type"]
            elif callable(hf_overrides):
                # If hf_overrides doesn't modify model_type, it will be passed straight
                # through and remain unchanged by this elif block
                dummy_model_type = f"dummy_{model_type}"
                dummy_kwargs = dict(architectures=[""], model_type=dummy_model_type)
                dummy_config = PretrainedConfig(**dummy_kwargs)
                dummy_model_type = hf_overrides(dummy_config).model_type
                model_type = dummy_model_type.removeprefix("dummy_")
199

200
        if model_type in _SPECULATIVE_DECODING_CONFIGS:
201
202
203
204
205
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
206
                trust_remote_code=trust_remote_code,
207
208
209
                **kwargs,
            )
        else:
210
211
212
213
214
215
            if model_type in _CONFIG_REGISTRY:
                # Register the config class to AutoConfig to ensure it's used in future
                # calls to `from_pretrained`
                config_class = _CONFIG_REGISTRY[model_type]
                config_class.model_type = model_type
                AutoConfig.register(model_type, config_class, exist_ok=True)
216
217
218
219
220
221
222
223
224
225
                # If the on-disk model_type differs from the overridden
                # one, register under both so AutoConfig.from_pretrained
                # returns the correct class regardless of what the
                # checkpoint says
                if (
                    config_model_type := config_dict.get("model_type")
                ) and config_model_type != model_type:
                    config_class.model_type = config_model_type
                    AutoConfig.register(config_model_type, config_class, exist_ok=True)
                    config_class.model_type = model_type
226
227
                # Now that it is registered, it is not considered remote code anymore
                trust_remote_code = False
228
            try:
229
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
230
231
232
233
234
235
236
237
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    **kwargs,
                )
            except ValueError as e:
238
239
240
241
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
242
243
244
245
246
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
247
248
                        "`--trust-remote-code` flag in the CLI."
                    )
249
250
251
252
253
254
255
256
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
257
258
    def parse(
        self,
259
        model: str | Path,
260
        trust_remote_code: bool,
261
262
        revision: str | None = None,
        code_revision: str | None = None,
263
264
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
265
266
267
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
268
269
270
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
271
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
272
273
                model, revision, **kwargs
            )
274
275
276
277
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

278
279
280
281
282
283
        # Get missing fields from HF config if available
        try:
            hf_config_dict, _ = PretrainedConfig.get_config_dict(
                model,
                revision=revision,
                code_revision=code_revision,
284
                **without_trust_remote_code(kwargs),
285
286
287
288
            )
        except OSError:  # Not found
            hf_config_dict = {}

Julien Denize's avatar
Julien Denize committed
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
        if config_dict.get("dtype") is None:
            with _mistral_patch_hf_hub_constants():
                model_str = model if isinstance(model, str) else model.as_posix()
                param_mt = get_safetensors_params_metadata(model_str, revision=revision)
            if param_mt:
                param_dtypes: set[torch.dtype] = {
                    _SAFETENSORS_TO_TORCH_DTYPE[dtype]
                    for info in param_mt.values()
                    if (dtype := info.get("dtype", None))
                    and dtype in _SAFETENSORS_TO_TORCH_DTYPE
                }

                if param_dtypes:
                    config_dict["dtype"] = common_broadcastable_dtype(param_dtypes)
                    logger.info_once(
                        "Inferred from consolidated*.safetensors files "
                        f"{config_dict['dtype']} dtype."
                    )

308
        config = adapt_config_dict(config_dict, defaults=hf_config_dict)
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
350
351
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
352
353
354
355
356
357
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
358
359
360
361
362
363
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
364
365
366
367
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
368
        if not issubclass(config_parser_cls, ConfigParserBase):
369
370
371
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
372
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
373
374
375
376
377
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
378
379
380
        return config_parser_cls

    return _wrapper
381
382


383
384
385
386
387
388
389
390
391
392
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
    """Some models may have no rope_theta in their config but still use RoPE.
    This function sets a default rope_theta if it's missing."""
    if getattr(config, "rope_parameters", None) is None:
        config.rope_parameters = {"rope_type": "default"}
    if "rope_theta" not in config.rope_parameters:
        config.rope_parameters["rope_theta"] = default_theta


def patch_rope_parameters(config: PretrainedConfig) -> None:
393
    """Provide backwards compatibility for RoPE."""
394
395
    from vllm.config.utils import getattr_iter

396
397
398
399
400
401
    # Older custom models may use non-standard field names
    # which need patching for both Transformers v4 and v5.
    names = ["rope_theta", "rotary_emb_base"]
    rope_theta = getattr_iter(config, names, None, warn=True)
    names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"]
    partial_rotary_factor = getattr_iter(config, names, None, warn=True)
402
    ompe = getattr(config, "original_max_position_embeddings", None)
403

404
    if Version(version("transformers")) < Version("5.0.0"):
405
        # Transformers v4 installed, legacy config fields may be present
406
407
        if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
            config.rope_parameters = rope_scaling
408
        if (
409
410
411
            rope_theta is not None
            or partial_rotary_factor is not None
            or ompe is not None
412
413
        ) and not getattr(config, "rope_parameters", None):
            config.rope_parameters = {"rope_type": "default"}
414
        # Patch legacy fields into rope_parameters
415
        if rope_theta is not None:
416
            config.rope_parameters["rope_theta"] = rope_theta
417
418
        if partial_rotary_factor is not None:
            config.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
419
420
        if ompe is not None:
            config.rope_parameters["original_max_position_embeddings"] = ompe
421
    elif rope_theta is not None or getattr(config, "rope_parameters", None):
422
        # Transformers v5 installed
423
424
425
426
427
428
        # Patch these fields in case they used non-standard names
        if rope_theta is not None:
            config.rope_theta = rope_theta
        if partial_rotary_factor is not None:
            config.partial_rotary_factor = partial_rotary_factor
        # Standardize and validate RoPE parameters
429
430
        config.standardize_rope_params()
        config.validate_rope()
431
432

    # No RoPE parameters to patch
433
    if getattr(config, "rope_parameters", None) is None:
434
435
436
        return

    # Handle nested rope_parameters in interleaved sliding attention models
437
    if is_rope_parameters_nested(config.rope_parameters):
438
        for rope_parameters_layer_type in config.rope_parameters.values():
439
440
            patch_rope_parameters_dict(rope_parameters_layer_type)
    else:
441
        patch_rope_parameters_dict(config.rope_parameters)
442
443


444
445
446
447
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
    if "rope_type" in rope_parameters and "type" in rope_parameters:
        rope_type = rope_parameters["rope_type"]
        rope_type_legacy = rope_parameters["type"]
448
449
450
451
452
        if (rope_type_legacy == "su" and rope_type == "longrope") or (
            rope_type_legacy == "mrope" and rope_type == "default"
        ):
            pass  # No action needed
        elif rope_type != rope_type_legacy:
453
454
455
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
456
457
                "You should only specify one of them."
            )
458

459
460
    if "rope_type" not in rope_parameters and "type" in rope_parameters:
        rope_parameters["rope_type"] = rope_parameters["type"]
461
462
        logger.info("Replacing legacy 'type' key with 'rope_type'")

463
464
    if "rope_type" not in rope_parameters:
        raise ValueError("rope_parameters should have a 'rope_type' key")
465

466
467
    if rope_parameters["rope_type"] == "su":
        rope_parameters["rope_type"] = "longrope"
468
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
469
    elif rope_parameters["rope_type"] == "mrope":
470
471
472
473
        if "mrope_section" not in rope_parameters:
            raise ValueError(
                "Legacy rope_type 'mrope' requires 'mrope_section' in rope_parameters"
            )
474
        rope_parameters["rope_type"] = "default"
475
476
477
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


478
def _uses_mrope(config: PretrainedConfig) -> bool:
479
480
    rope_parameters = getattr(config, "rope_parameters", None)
    if rope_parameters is None:
481
482
        return False

483
    return "mrope_section" in rope_parameters
484
485


486
487
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
488
489
490
491
492
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
def uses_xdrope_dim(config: PretrainedConfig) -> int:
    """Detect if the model with this config uses XD-ROPE."""
    xdrope_section = getattr(config, "xdrope_section", None)
    if xdrope_section is not None and isinstance(xdrope_section, list):
        return len(xdrope_section)
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return 0

    if isinstance(rope_scaling, dict) and "xdrope_section" in rope_scaling:
        xdrope_section = rope_scaling["xdrope_section"]
        if xdrope_section is not None and isinstance(xdrope_section, list):
            return len(xdrope_section)

    return 0


525
526
527
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

528
529
530
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

531
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
532
533


534
535
536
537
538
539
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
540
        return len(set(layer_types)) > 1
541
542
543
    return False


544
545
546
547
548
549
550
551
552
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


553
554
555
556
557
558
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
559
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
560
561
562
    return config


563
def maybe_override_with_speculators(
564
    model: str,
565
    tokenizer: str | None,
566
    trust_remote_code: bool,
567
568
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
569
    hf_token: bool | str | None = None,
570
    **kwargs,
571
) -> tuple[str, str | None, dict[str, Any] | None]:
572
    """
573
574
575
576
577
578
579
580
581
582
583
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config
584
        hf_token: HuggingFace token for authenticated model access
585
586
587

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
588
    """
589
    if check_gguf_file(model):
590
591
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
592
593
594
    elif is_remote_gguf(model):
        repo_id, _ = split_remote_gguf(model)
        gguf_model_repo = Path(repo_id)
595
596
    else:
        gguf_model_repo = None
597
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
598
    config_dict, _ = PretrainedConfig.get_config_dict(
599
        model if gguf_model_repo is None else gguf_model_repo,
600
        revision=revision,
601
        token=hf_token,
602
        **without_trust_remote_code(kwargs),
603
    )
604
605
606
607
608
609
610
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
611
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
612

613
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
614
615
        config_dict=config_dict
    )
616
617

    # Set the draft model to the speculators model
618
    speculative_config["model"] = model
619
620
621
622
623

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

624
    return model, tokenizer, speculative_config
625
626


627
def get_config(
628
    model: str | Path,
629
    trust_remote_code: bool,
630
631
632
633
634
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
635
636
637
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
638

639
640
641
642
643
644
645
646
647
648
649
650
    _is_gguf = is_gguf(model)
    _is_remote_gguf = is_remote_gguf(model)
    if _is_gguf:
        if check_gguf_file(model):
            # Local GGUF file
            kwargs["gguf_file"] = Path(model).name
            model = Path(model).parent
        elif _is_remote_gguf:
            # Remote GGUF - extract repo_id from repo_id:quant_type format
            # The actual GGUF file will be downloaded later by GGUFModelLoader
            # Keep model as repo_id:quant_type for download, but use repo_id for config
            model, _ = split_remote_gguf(model)
651

652
    if config_format == "auto":
653
        try:
654
655
            # First check for Mistral to avoid defaulting to
            # Transformers implementation.
656
657
658
659
660
            if is_mistral_model_repo(
                model_name_or_path=str(model), revision=revision
            ) and file_or_path_exists(
                model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
            ):
661
                config_format = "mistral"
662
            elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
663
664
665
                model, HF_CONFIG_NAME, revision=revision
            ):
                config_format = "hf"
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
            # Remote GGUF models must have config.json in repo,
            # otherwise the config can't be parsed correctly.
            # FIXME(Isotr0py): Support remote GGUF repos without config.json
            elif _is_remote_gguf and not file_or_path_exists(
                model, HF_CONFIG_NAME, revision=revision
            ):
                err_msg = (
                    "Could not find config.json for remote GGUF model repo. "
                    "To load remote GGUF model through `<repo_id>:<quant_type>`, "
                    "ensure your model has config.json (HF format) file. "
                    "Otherwise please specify --hf-config-path <original_repo> "
                    "in engine args to fetch config from unquantized hf model."
                )
                logger.error(err_msg)
                raise ValueError(err_msg)
681
682
683
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
684
685
686
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
687
688
                    "in engine args for customized config parser."
                )
689
690
691
692
693
694
695
696
697
698
699

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
700
                "'params.json'.\n"
701
            ).format(model=model)
702
703

            raise ValueError(error_message) from e
704

705
706
707
708
709
710
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
711
        hf_overrides=hf_overrides_kw or hf_overrides_fn,
712
713
        **kwargs,
    )
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735

    # Patching defaults for GGUF models
    if _is_gguf:
        # Some models have different default values between GGUF and HF.
        def apply_gguf_default(key: str, gguf_default: Any):
            """
            Apply GGUF defaults unless explicitly configured.

            This function reads/writes external `config` and `config_dict`.
            If the specified `key` is not in `config_dict` (i.e. not explicitly
            configured and the default HF value is used), it updates the
            corresponding `config` value to `gguf_default`.
            """
            if key not in config_dict:
                config.update({key: gguf_default})

        # Apply architecture-specific GGUF defaults.
        if config.model_type in {"qwen3_moe"}:
            # Qwen3 MoE: norm_topk_prob is always true.
            # Note that, this parameter is always false (HF default) on Qwen2 MoE.
            apply_gguf_default("norm_topk_prob", True)

736
    # Special architecture mapping check for GGUF models
737
    if _is_gguf:
738
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
739
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
740
741
742
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

743
744
745
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
746
747
748
749
750
751
752
753
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
754

755
756
757
758
759
760
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
761
762
763
764
765
766
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
767
768
769

    if quantization_config is not None:
        config.quantization_config = quantization_config
770
        # auto-enable DeepGEMM UE8M0 if model config requests it
771
        scale_fmt = quantization_config.get("scale_fmt", None)
772
        if scale_fmt in ("ue8m0",):
773
774
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
775
                logger.info_once(
776
777
                    (
                        "Detected quantization_config.scale_fmt=%s; "
778
                        "enabling UE8M0 for DeepGEMM."
779
                    ),
780
781
                    scale_fmt,
                )
782
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
783
                logger.warning_once(
784
785
786
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
787
788
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
789
                    ),
790
791
                    scale_fmt,
                )
792

793
794
795
796
797
798
799
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

800
801
802
803
804
805
806
807
    # Exhaustively patch RoPE parameters everywhere they might be
    patch_rope_parameters(config)
    patch_rope_parameters(config.get_text_config())
    SubConfigs: TypeAlias = dict[str, PretrainedConfig]
    sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
    if sub_configs:
        for sub_config in sub_configs:
            patch_rope_parameters(getattr(config, sub_config))
808

809
810
811
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

812
    return config
813
814


815
@cache
816
817
818
819
def get_pooling_config(
    model: str,
    revision: str | None = "main",
) -> dict[str, Any] | None:
820
    """
821
822
823
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
824
825

    Args:
826
        model: The name of the Hugging Face model.
827
        revision: The specific version of the model to use.
828
            Defaults to 'main'.
829
830

    Returns:
831
        A dictionary containing the pooling type and whether
832
            normalization is used, or None if no pooling configuration is found.
833
    """
834
835
    if is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
836
837

    modules_file_name = "modules.json"
838
839

    modules_dict = None
840
841
842
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
843
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
844
845
846
847

    if modules_dict is None:
        return None

848
849
    logger.info("Found sentence-transformers modules configuration.")

850
851
852
853
854
855
856
857
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
858
    normalize = bool(
859
860
861
862
863
864
865
866
867
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
868
869

    if pooling:
870
        from vllm.config.pooler import SEQ_POOLING_TYPES, TOK_POOLING_TYPES
871

872
873
        pooling_file_name = "{}/config.json".format(pooling["path"])
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision) or {}
874

875
        logger.info("Found pooling configuration.")
876

877
        config: dict[str, Any] = {"use_activation": normalize}
878
879
880
881
882
883
884
885
886
887
888
        for key, val in pooling_dict.items():
            if val is True:
                pooling_type = parse_pooling_type(key)
                if pooling_type in SEQ_POOLING_TYPES:
                    config["seq_pooling_type"] = pooling_type
                elif pooling_type in TOK_POOLING_TYPES:
                    config["tok_pooling_type"] = pooling_type
                else:
                    logger.debug("Skipping unrelated field: %r=%r", key, val)

        return config
889
890
891
892

    return None


893
def parse_pooling_type(pooling_name: str):
894
895
896
897
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
898
        pooling_name = pooling_name.split("_", 1)[0]
899
900
901
902

    if "lasttoken" in pooling_name:
        pooling_name = "last"

903
    return pooling_name.upper()
904
905


906
@cache
907
def get_sentence_transformer_tokenizer_config(
908
    model: str | Path, revision: str | None = "main"
909
) -> dict[str, Any] | None:
910
    """
911
    Returns the tokenization configuration dictionary for a
912
913
914
    given Sentence Transformer BERT model.

    Parameters:
915
    - model (str|Path): The name of the Sentence Transformer
916
917
918
919
920
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
921
    - dict: A dictionary containing the configuration parameters
922
923
    for the Sentence Transformer BERT model.
    """
924
925
926
927
928
929
930
931
932
933
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
934
935

    for config_file in sentence_transformer_config_files:
936
937
938
939
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
940
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
941
942
            if encoder_dict:
                break
943

944
    if not encoder_dict and not Path(model).is_absolute():
945
946
        try:
            # If model is on HuggingfaceHub, get the repo files
947
            repo_files = list_repo_files(model, revision=revision)
948
        except Exception:
949
950
951
952
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
953
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
954
955
956
                if encoder_dict:
                    break

957
958
959
    if not encoder_dict:
        return None

960
961
    logger.info("Found sentence-transformers tokenize configuration.")

962
963
964
965
966
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


967
def maybe_register_config_serialize_by_value() -> None:
968
969
    """Try to register HF model configuration class to serialize by value

970
971
972
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
973

974
    Examples:
975

976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
1000
1001
    try:
        import transformers_modules
1002

1003
        transformers_modules_available = True
1004
    except ImportError:
1005
        transformers_modules_available = False
1006
1007
1008
1009
1010

    try:
        import multiprocessing
        import pickle

1011
1012
        import cloudpickle

1013
        from vllm.config import VllmConfig
1014

1015
1016
1017
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
1018
        def _reduce_config(config: VllmConfig):
1019
            return (pickle.loads, (cloudpickle.dumps(config),))
1020

1021
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
1022

1023
1024
1025
1026
1027
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
1028
            from vllm.v1.executor.ray_utils import ray
1029

1030
1031
1032
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

1033
1034
1035
1036
1037
1038
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
1039
1040
            exc_info=e,
        )
1041
1042


1043
def get_hf_image_processor_config(
1044
1045
1046
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
1047
    **kwargs,
1048
) -> dict[str, Any]:
1049
    # ModelScope does not provide an interface for image_processor
1050
    if envs.VLLM_USE_MODELSCOPE:
1051
        return dict()
1052
    # Separate model folder from file path for GGUF models
1053
    if check_gguf_file(model):
1054
        model = Path(model).parent
1055
1056
    elif is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
1057
1058
1059
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
1060
1061


1062
1063
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
1064
    No op for pure text models.
1065
    """
1066
1067
    text_config = config.get_text_config()

1068
1069
1070
1071
1072
1073
1074
    if text_config is not config and not hasattr(text_config, "num_attention_heads"):
        raise ValueError(
            "The text_config extracted from the model config does not have "
            "`num_attention_heads` attribute. This indicates a mismatch "
            "between the model config and vLLM's expectations. Please "
            "ensure that the model config is compatible with vLLM."
        )
1075
1076

    return text_config
1077
1078
1079
1080
1081


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
1082
1083
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
1084
    hf_token: bool | str | None = None,
1085
) -> GenerationConfig | None:
1086
1087
1088
1089
1090
1091
1092
    # GGUF files don't have generation_config.json - their config is embedded
    # in the file header. Skip all filesystem lookups to avoid re-reading the
    # memory-mapped file, which can hang in multi-process scenarios when the
    # EngineCore process already has the file mapped.
    if is_gguf(model):
        return None

1093
1094
1095
1096
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
1097
            token=hf_token,
1098
1099
1100
1101
1102
1103
1104
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1105
                config_format=config_format,
1106
                token=hf_token,
1107
1108
1109
1110
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1111
1112


1113
1114
1115
def try_get_safetensors_metadata(
    model: str,
    *,
1116
    revision: str | None = None,
1117
1118
):
    get_safetensors_metadata_partial = partial(
1119
        get_safetensors_metadata, model, revision=revision
1120
1121
1122
    )

    try:
1123
1124
1125
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1126
1127
    except Exception:
        return None
1128
1129
1130


def try_get_tokenizer_config(
1131
    pretrained_model_name_or_path: str | os.PathLike,
1132
    trust_remote_code: bool,
1133
1134
    revision: str | None = None,
) -> dict[str, Any] | None:
1135
1136
1137
1138
1139
1140
1141
1142
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1143
1144


1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

1158
1159
1160
1161
1162
        _DENSE_MODULE_TYPES = {
            "sentence_transformers.models.Dense",
            "pylate.models.Dense.Dense",
        }
        dense_modules = [m for m in modules if m.get("type") in _DENSE_MODULE_TYPES]
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1181
1182
1183
def get_safetensors_params_metadata(
    model: str,
    *,
1184
    revision: str | None = None,
1185
1186
) -> dict[str, Any]:
    """
1187
    Get the safetensors parameters metadata for remote/local model repository.
1188
1189
1190
1191
1192
1193
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1194
1195
1196
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1209
1210
1211
1212
1213
1214
1215
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1216
1217
            f"and if the config file exists."
        )
1218
1219
1220
1221
1222
1223
1224
1225
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1226
1227
1228
1229
1230
1231
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1232
1233
1234
1235
1236
1237
1238
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1239
1240
            exc_info=e,
        )
1241
1242

    return max_position_embeddings