config.py 41.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
5
from collections.abc import Callable
6
from dataclasses import asdict
7
from functools import cache, partial
8
from importlib.metadata import version
9
from pathlib import Path
10
from typing import Any, Literal, TypeAlias
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
14
15
from huggingface_hub import (
    get_safetensors_metadata,
)
16
from packaging.version import Version
17
from transformers import GenerationConfig, PretrainedConfig
18
from transformers.models.auto.image_processing_auto import get_image_processor_config
19
20
21
22
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
23
from transformers.models.auto.tokenization_auto import get_tokenizer_config
24
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
25

26
from vllm import envs
27
from vllm.logger import init_logger
28
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
29
30

from .config_parser_base import ConfigParserBase
31
32
33
34
35
36
from .gguf_utils import (
    check_gguf_file,
    is_gguf,
    is_remote_gguf,
    split_remote_gguf,
)
37
from .repo_utils import (
38
39
40
41
42
43
44
    _get_hf_token,
    file_or_path_exists,
    get_hf_file_to_dict,
    list_repo_files,
    try_get_local_file,
    with_retry,
)
45

46
47
48
49
50
51
52
53
54
55
try:
    # Transformers v5
    from transformers.configuration_utils import ALLOWED_ATTENTION_LAYER_TYPES
except ImportError:
    # Transformers v4
    from transformers.configuration_utils import (
        ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
    )


56
if envs.VLLM_USE_MODELSCOPE:
57
58
59
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
60

61
62
MISTRAL_CONFIG_NAME = "params.json"

63
64
logger = init_logger(__name__)

65

66
67
class LazyConfigDict(dict):
    def __getitem__(self, key):
68
69
70
        if isinstance(value := super().__getitem__(key), type):
            return value

71
        import vllm.transformers_utils.configs as configs
72

73
        return getattr(configs, value)
74
75
76


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
77
    afmoe="AfmoeConfig",
78
    bagel="BagelConfig",
79
80
    chatglm="ChatGLMConfig",
    deepseek_vl_v2="DeepseekVLV2Config",
81
    deepseek_v32="DeepseekV3Config",
82
    flex_olmo="FlexOlmoConfig",
83
    hunyuan_vl="HunYuanVLConfig",
oscardev256's avatar
oscardev256 committed
84
    isaac="IsaacConfig",
85
    kimi_linear="KimiLinearConfig",
86
87
88
89
90
91
    kimi_vl="KimiVLConfig",
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
92
    midashenglm="MiDashengLMConfig",
93
94
95
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
96
    olmo3="Olmo3Config",
97
98
99
100
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
101
    qwen3_next="Qwen3NextConfig",
Paul Pak's avatar
Paul Pak committed
102
    lfm2_moe="Lfm2MoeConfig",
103
    tarsier2="Tarsier2Config",
104
)
105

106
107
108
109
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

110
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
111
    "internvl_chat": {"has_no_defaults_at_init": True},
112
    "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"},
113
    "NVLM_D": {"has_no_defaults_at_init": True},
114
115
}

116

117
118
119
120
121
122
123
124
def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool:
    """Check if rope_parameters is nested by layer types."""
    # Cannot be nested if rope_parameters is empty
    if not rope_parameters:
        return False
    return set(rope_parameters.keys()).issubset(ALLOWED_ATTENTION_LAYER_TYPES)


125
class HFConfigParser(ConfigParserBase):
126
127
    def parse(
        self,
128
        model: str | Path,
129
        trust_remote_code: bool,
130
131
        revision: str | None = None,
        code_revision: str | None = None,
132
133
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
134
135
136
137
138
139
140
141
142
143
144
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            token=_get_hf_token(),
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
145
146
147
148
149
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
150
151
152
        # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
        if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
            model_type = hf_overrides.get("model_type", model_type)
153
154
155
156
157
158
159
160
161
162
163
164

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        else:
            try:
165
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
166
167
168
169
170
171
172
173
174
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    token=_get_hf_token(),
                    **kwargs,
                )
            except ValueError as e:
175
176
177
178
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
179
180
181
182
183
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
184
185
                        "`--trust-remote-code` flag in the CLI."
                    )
186
187
188
189
190
191
192
193
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
194
195
    def parse(
        self,
196
        model: str | Path,
197
        trust_remote_code: bool,
198
199
        revision: str | None = None,
        code_revision: str | None = None,
200
201
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
202
203
204
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
205
206
207
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
208
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
209
210
                model, revision, **kwargs
            )
211
212
213
214
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

215
216
217
218
219
220
221
222
223
224
225
226
227
        # Get missing fields from HF config if available
        try:
            hf_config_dict, _ = PretrainedConfig.get_config_dict(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        except OSError:  # Not found
            hf_config_dict = {}

        config = adapt_config_dict(config_dict, defaults=hf_config_dict)
228
229
230

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
231
232
233
        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
            sliding_window, list
        ):
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
282
283
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
284
285
286
287
288
289
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
290
291
292
293
294
295
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
296
297
298
299
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
300
        if not issubclass(config_parser_cls, ConfigParserBase):
301
302
303
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
304
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
305
306
307
308
309
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
310
311
312
        return config_parser_cls

    return _wrapper
313
314


315
316
317
318
319
320
321
322
323
324
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
    """Some models may have no rope_theta in their config but still use RoPE.
    This function sets a default rope_theta if it's missing."""
    if getattr(config, "rope_parameters", None) is None:
        config.rope_parameters = {"rope_type": "default"}
    if "rope_theta" not in config.rope_parameters:
        config.rope_parameters["rope_theta"] = default_theta


def patch_rope_parameters(config: PretrainedConfig) -> None:
325
    """Provide backwards compatibility for RoPE."""
326
327
    from vllm.config.utils import getattr_iter

328
329
330
331
332
333
    # Older custom models may use non-standard field names
    # which need patching for both Transformers v4 and v5.
    names = ["rope_theta", "rotary_emb_base"]
    rope_theta = getattr_iter(config, names, None, warn=True)
    names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"]
    partial_rotary_factor = getattr_iter(config, names, None, warn=True)
334
    ompe = getattr(config, "original_max_position_embeddings", None)
335

336
337
    if Version(version("transformers")) < Version("5.0.0.dev0"):
        # Transformers v4 installed, legacy config fields may be present
338
339
        if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
            config.rope_parameters = rope_scaling
340
        if (
341
342
343
            rope_theta is not None
            or partial_rotary_factor is not None
            or ompe is not None
344
345
        ) and not getattr(config, "rope_parameters", None):
            config.rope_parameters = {"rope_type": "default"}
346
        # Patch legacy fields into rope_parameters
347
        if rope_theta is not None:
348
            config.rope_parameters["rope_theta"] = rope_theta
349
350
        if partial_rotary_factor is not None:
            config.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
351
352
        if ompe is not None:
            config.rope_parameters["original_max_position_embeddings"] = ompe
353
    elif rope_theta is not None or getattr(config, "rope_parameters", None):
354
        # Transformers v5 installed
355
356
357
358
359
360
        # Patch these fields in case they used non-standard names
        if rope_theta is not None:
            config.rope_theta = rope_theta
        if partial_rotary_factor is not None:
            config.partial_rotary_factor = partial_rotary_factor
        # Standardize and validate RoPE parameters
361
362
        config.standardize_rope_params()
        config.validate_rope()
363
364

    # No RoPE parameters to patch
365
    if getattr(config, "rope_parameters", None) is None:
366
367
368
        return

    # Handle nested rope_parameters in interleaved sliding attention models
369
    if is_rope_parameters_nested(config.rope_parameters):
370
        for rope_parameters_layer_type in config.rope_parameters.values():
371
372
            patch_rope_parameters_dict(rope_parameters_layer_type)
    else:
373
        patch_rope_parameters_dict(config.rope_parameters)
374
375


376
377
378
379
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
    if "rope_type" in rope_parameters and "type" in rope_parameters:
        rope_type = rope_parameters["rope_type"]
        rope_type_legacy = rope_parameters["type"]
380
381
382
383
384
        if (rope_type_legacy == "su" and rope_type == "longrope") or (
            rope_type_legacy == "mrope" and rope_type == "default"
        ):
            pass  # No action needed
        elif rope_type != rope_type_legacy:
385
386
387
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
388
389
                "You should only specify one of them."
            )
390

391
392
    if "rope_type" not in rope_parameters and "type" in rope_parameters:
        rope_parameters["rope_type"] = rope_parameters["type"]
393
394
        logger.info("Replacing legacy 'type' key with 'rope_type'")

395
396
    if "rope_type" not in rope_parameters:
        raise ValueError("rope_parameters should have a 'rope_type' key")
397

398
399
    if rope_parameters["rope_type"] == "su":
        rope_parameters["rope_type"] = "longrope"
400
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
401
    elif rope_parameters["rope_type"] == "mrope":
402
403
404
405
        if "mrope_section" not in rope_parameters:
            raise ValueError(
                "Legacy rope_type 'mrope' requires 'mrope_section' in rope_parameters"
            )
406
        rope_parameters["rope_type"] = "default"
407
408
409
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


410
def _uses_mrope(config: PretrainedConfig) -> bool:
411
412
    rope_parameters = getattr(config, "rope_parameters", None)
    if rope_parameters is None:
413
414
        return False

415
    return "mrope_section" in rope_parameters
416
417


418
419
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
420
421
422
423
424
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
def uses_xdrope_dim(config: PretrainedConfig) -> int:
    """Detect if the model with this config uses XD-ROPE."""
    xdrope_section = getattr(config, "xdrope_section", None)
    if xdrope_section is not None and isinstance(xdrope_section, list):
        return len(xdrope_section)
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return 0

    if isinstance(rope_scaling, dict) and "xdrope_section" in rope_scaling:
        xdrope_section = rope_scaling["xdrope_section"]
        if xdrope_section is not None and isinstance(xdrope_section, list):
            return len(xdrope_section)

    return 0


457
458
459
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

460
461
462
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

463
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
464
465


466
467
468
469
470
471
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
472
        return len(set(layer_types)) > 1
473
474
475
    return False


476
477
478
479
480
481
482
483
484
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


485
486
487
488
489
490
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
491
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
492
493
494
    return config


495
def maybe_override_with_speculators(
496
    model: str,
497
    tokenizer: str | None,
498
    trust_remote_code: bool,
499
500
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
501
    **kwargs,
502
) -> tuple[str, str | None, dict[str, Any] | None]:
503
    """
504
505
506
507
508
509
510
511
512
513
514
515
516
517
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
518
    """
519
    if check_gguf_file(model):
520
521
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
522
523
524
    elif is_remote_gguf(model):
        repo_id, _ = split_remote_gguf(model)
        gguf_model_repo = Path(repo_id)
525
526
    else:
        gguf_model_repo = None
527
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
528
    config_dict, _ = PretrainedConfig.get_config_dict(
529
        model if gguf_model_repo is None else gguf_model_repo,
530
531
532
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
533
        **kwargs,
534
    )
535
536
537
538
539
540
541
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
542
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
543

544
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
545
546
        config_dict=config_dict
    )
547
548

    # Set the draft model to the speculators model
549
    speculative_config["model"] = model
550
551
552
553
554

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

555
    return model, tokenizer, speculative_config
556
557


558
def get_config(
559
    model: str | Path,
560
    trust_remote_code: bool,
561
562
563
564
565
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
566
567
568
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
569

570
571
572
573
574
575
576
577
578
579
580
581
    _is_gguf = is_gguf(model)
    _is_remote_gguf = is_remote_gguf(model)
    if _is_gguf:
        if check_gguf_file(model):
            # Local GGUF file
            kwargs["gguf_file"] = Path(model).name
            model = Path(model).parent
        elif _is_remote_gguf:
            # Remote GGUF - extract repo_id from repo_id:quant_type format
            # The actual GGUF file will be downloaded later by GGUFModelLoader
            # Keep model as repo_id:quant_type for download, but use repo_id for config
            model, _ = split_remote_gguf(model)
582

583
    if config_format == "auto":
584
        try:
585
586
587
            # First check for Mistral to avoid defaulting to
            # Transformers implementation.
            if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
588
                config_format = "mistral"
589
            elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
590
591
592
                model, HF_CONFIG_NAME, revision=revision
            ):
                config_format = "hf"
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
            # Remote GGUF models must have config.json in repo,
            # otherwise the config can't be parsed correctly.
            # FIXME(Isotr0py): Support remote GGUF repos without config.json
            elif _is_remote_gguf and not file_or_path_exists(
                model, HF_CONFIG_NAME, revision=revision
            ):
                err_msg = (
                    "Could not find config.json for remote GGUF model repo. "
                    "To load remote GGUF model through `<repo_id>:<quant_type>`, "
                    "ensure your model has config.json (HF format) file. "
                    "Otherwise please specify --hf-config-path <original_repo> "
                    "in engine args to fetch config from unquantized hf model."
                )
                logger.error(err_msg)
                raise ValueError(err_msg)
608
609
610
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
611
612
613
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
614
615
                    "in engine args for customized config parser."
                )
616
617
618
619
620
621
622
623
624
625
626

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
627
                "'params.json'.\n"
628
            ).format(model=model)
629
630

            raise ValueError(error_message) from e
631

632
633
634
635
636
637
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
638
        hf_overrides=hf_overrides_kw,
639
640
        **kwargs,
    )
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662

    # Patching defaults for GGUF models
    if _is_gguf:
        # Some models have different default values between GGUF and HF.
        def apply_gguf_default(key: str, gguf_default: Any):
            """
            Apply GGUF defaults unless explicitly configured.

            This function reads/writes external `config` and `config_dict`.
            If the specified `key` is not in `config_dict` (i.e. not explicitly
            configured and the default HF value is used), it updates the
            corresponding `config` value to `gguf_default`.
            """
            if key not in config_dict:
                config.update({key: gguf_default})

        # Apply architecture-specific GGUF defaults.
        if config.model_type in {"qwen3_moe"}:
            # Qwen3 MoE: norm_topk_prob is always true.
            # Note that, this parameter is always false (HF default) on Qwen2 MoE.
            apply_gguf_default("norm_topk_prob", True)

663
    # Special architecture mapping check for GGUF models
664
    if _is_gguf:
665
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
666
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
667
668
669
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

670
671
672
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
673
674
675
676
677
678
679
680
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
681

682
683
684
685
686
687
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
688
689
690
691
692
693
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
694
695
696

    if quantization_config is not None:
        config.quantization_config = quantization_config
697
        # auto-enable DeepGEMM UE8M0 if model config requests it
698
        scale_fmt = quantization_config.get("scale_fmt", None)
699
        if scale_fmt in ("ue8m0",):
700
701
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
702
                logger.info_once(
703
704
                    (
                        "Detected quantization_config.scale_fmt=%s; "
705
                        "enabling UE8M0 for DeepGEMM."
706
                    ),
707
708
                    scale_fmt,
                )
709
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
710
                logger.warning_once(
711
712
713
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
714
715
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
716
                    ),
717
718
                    scale_fmt,
                )
719

720
721
722
723
724
725
726
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

727
728
729
730
731
732
733
734
    # Exhaustively patch RoPE parameters everywhere they might be
    patch_rope_parameters(config)
    patch_rope_parameters(config.get_text_config())
    SubConfigs: TypeAlias = dict[str, PretrainedConfig]
    sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
    if sub_configs:
        for sub_config in sub_configs:
            patch_rope_parameters(getattr(config, sub_config))
735

736
737
738
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

739
    return config
740
741


742
@cache
743
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None:
744
    """
745
746
747
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
748
749

    Args:
750
        model: The name of the Hugging Face model.
751
        revision: The specific version of the model to use.
752
            Defaults to 'main'.
753
754

    Returns:
755
        A dictionary containing the pooling type and whether
756
            normalization is used, or None if no pooling configuration is found.
757
    """
758
759
    if is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
760
761

    modules_file_name = "modules.json"
762
763

    modules_dict = None
764
765
766
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
767
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
768
769
770
771

    if modules_dict is None:
        return None

772
773
    logger.info("Found sentence-transformers modules configuration.")

774
775
776
777
778
779
780
781
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
782
    normalize = bool(
783
784
785
786
787
788
789
790
791
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
792
793
794

    if pooling:
        pooling_file_name = "{}/config.json".format(pooling["path"])
795
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
796
        pooling_type_name = next(
797
798
            (item for item, val in pooling_dict.items() if val is True), None
        )
799
800
801
802

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

803
        logger.info("Found pooling configuration.")
804
805
806
807
808
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


809
def get_pooling_config_name(pooling_name: str) -> str | None:
810
811
812
813
814
815
816
817
818
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

819
    supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"]
820
821
    pooling_type_name = pooling_name.upper()

822
823
824
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

825
    raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
826
827


828
@cache
829
def get_sentence_transformer_tokenizer_config(
830
    model: str | Path, revision: str | None = "main"
831
):
832
    """
833
    Returns the tokenization configuration dictionary for a
834
835
836
    given Sentence Transformer BERT model.

    Parameters:
837
    - model (str|Path): The name of the Sentence Transformer
838
839
840
841
842
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
843
    - dict: A dictionary containing the configuration parameters
844
845
    for the Sentence Transformer BERT model.
    """
846
847
848
849
850
851
852
853
854
855
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
856
857

    for config_file in sentence_transformer_config_files:
858
859
860
861
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
862
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
863
864
            if encoder_dict:
                break
865

866
    if not encoder_dict and not Path(model).is_absolute():
867
868
        try:
            # If model is on HuggingfaceHub, get the repo files
869
870
871
            repo_files = list_repo_files(
                model, revision=revision, token=_get_hf_token()
            )
872
        except Exception:
873
874
875
876
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
877
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
878
879
880
                if encoder_dict:
                    break

881
882
883
    if not encoder_dict:
        return None

884
885
    logger.info("Found sentence-transformers tokenize configuration.")

886
887
888
889
890
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


891
def maybe_register_config_serialize_by_value() -> None:
892
893
    """Try to register HF model configuration class to serialize by value

894
895
896
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
897

898
    Examples:
899

900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
924
925
    try:
        import transformers_modules
926

927
        transformers_modules_available = True
928
    except ImportError:
929
        transformers_modules_available = False
930
931
932
933
934

    try:
        import multiprocessing
        import pickle

935
936
        import cloudpickle

937
        from vllm.config import VllmConfig
938

939
940
941
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
942
        def _reduce_config(config: VllmConfig):
943
            return (pickle.loads, (cloudpickle.dumps(config),))
944

945
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
946

947
948
949
950
951
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
952
            from vllm.v1.executor.ray_utils import ray
953

954
955
956
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

957
958
959
960
961
962
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
963
964
            exc_info=e,
        )
965
966


967
def get_hf_image_processor_config(
968
969
970
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
971
    **kwargs,
972
) -> dict[str, Any]:
973
    # ModelScope does not provide an interface for image_processor
974
    if envs.VLLM_USE_MODELSCOPE:
975
        return dict()
976
    # Separate model folder from file path for GGUF models
977
    if check_gguf_file(model):
978
        model = Path(model).parent
979
980
    elif is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
981
982
983
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
984
985


986
987
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
988
    No op for pure text models.
989
    """
990
991
    text_config = config.get_text_config()

992
993
994
995
996
997
998
    if text_config is not config and not hasattr(text_config, "num_attention_heads"):
        raise ValueError(
            "The text_config extracted from the model config does not have "
            "`num_attention_heads` attribute. This indicates a mismatch "
            "between the model config and vLLM's expectations. Please "
            "ensure that the model config is compatible with vLLM."
        )
999
1000

    return text_config
1001
1002
1003
1004
1005


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
1006
1007
1008
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
) -> GenerationConfig | None:
1009
1010
1011
1012
1013
1014
1015
    # GGUF files don't have generation_config.json - their config is embedded
    # in the file header. Skip all filesystem lookups to avoid re-reading the
    # memory-mapped file, which can hang in multi-process scenarios when the
    # EngineCore process already has the file mapped.
    if is_gguf(model):
        return None

1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1027
                config_format=config_format,
1028
1029
1030
1031
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1032
1033


1034
1035
1036
def try_get_safetensors_metadata(
    model: str,
    *,
1037
    revision: str | None = None,
1038
1039
1040
1041
1042
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
1043
        token=_get_hf_token(),
1044
1045
1046
    )

    try:
1047
1048
1049
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1050
1051
    except Exception:
        return None
1052
1053
1054


def try_get_tokenizer_config(
1055
    pretrained_model_name_or_path: str | os.PathLike,
1056
    trust_remote_code: bool,
1057
1058
    revision: str | None = None,
) -> dict[str, Any] | None:
1059
1060
1061
1062
1063
1064
1065
1066
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1067
1068


1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

        dense_modules = [
            m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
        ]
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1103
1104
1105
def get_safetensors_params_metadata(
    model: str,
    *,
1106
    revision: str | None = None,
1107
1108
1109
1110
1111
1112
1113
1114
1115
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1116
1117
1118
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1131
1132
1133
1134
1135
1136
1137
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1138
1139
            f"and if the config file exists."
        )
1140
1141
1142
1143
1144
1145
1146
1147
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1148
1149
1150
1151
1152
1153
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1154
1155
1156
1157
1158
1159
1160
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1161
1162
            exc_info=e,
        )
1163
1164

    return max_position_embeddings