config.py 40.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
5
from collections.abc import Callable
6
from dataclasses import asdict
7
from functools import cache, partial
8
from importlib.metadata import version
9
from pathlib import Path
10
from typing import Any, Literal, TypeAlias
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
14
15
from huggingface_hub import (
    get_safetensors_metadata,
)
16
from packaging.version import Version
17
from transformers import GenerationConfig, PretrainedConfig
18
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
19
from transformers.models.auto.image_processing_auto import get_image_processor_config
20
21
22
23
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
24
from transformers.models.auto.tokenization_auto import get_tokenizer_config
25
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26

27
from vllm import envs
28
from vllm.logger import init_logger
29
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
30
31

from .config_parser_base import ConfigParserBase
32
33
34
35
36
37
from .gguf_utils import (
    check_gguf_file,
    is_gguf,
    is_remote_gguf,
    split_remote_gguf,
)
38
from .repo_utils import (
39
40
41
42
43
44
45
    _get_hf_token,
    file_or_path_exists,
    get_hf_file_to_dict,
    list_repo_files,
    try_get_local_file,
    with_retry,
)
46

47
if envs.VLLM_USE_MODELSCOPE:
48
49
50
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
51

52
53
MISTRAL_CONFIG_NAME = "params.json"

54
55
logger = init_logger(__name__)

56

57
58
class LazyConfigDict(dict):
    def __getitem__(self, key):
59
60
61
        if isinstance(value := super().__getitem__(key), type):
            return value

62
        import vllm.transformers_utils.configs as configs
63

64
        return getattr(configs, value)
65
66
67


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
68
    afmoe="AfmoeConfig",
69
    bagel="BagelConfig",
70
71
    chatglm="ChatGLMConfig",
    deepseek_vl_v2="DeepseekVLV2Config",
72
    deepseek_v32="DeepseekV3Config",
73
    flex_olmo="FlexOlmoConfig",
74
    hunyuan_vl="HunYuanVLConfig",
75
    kimi_linear="KimiLinearConfig",
76
77
78
79
80
81
    kimi_vl="KimiVLConfig",
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
82
    midashenglm="MiDashengLMConfig",
83
84
85
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
86
    olmo3="Olmo3Config",
87
88
89
90
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
91
    qwen3_next="Qwen3NextConfig",
Paul Pak's avatar
Paul Pak committed
92
    lfm2_moe="Lfm2MoeConfig",
93
    tarsier2="Tarsier2Config",
94
)
95

96
97
98
99
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

100
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
101
    "internvl_chat": {"has_no_defaults_at_init": True},
102
    "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"},
103
    "NVLM_D": {"has_no_defaults_at_init": True},
104
105
}

106

107
class HFConfigParser(ConfigParserBase):
108
109
    def parse(
        self,
110
        model: str | Path,
111
        trust_remote_code: bool,
112
113
        revision: str | None = None,
        code_revision: str | None = None,
114
115
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
116
117
118
119
120
121
122
123
124
125
126
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            token=_get_hf_token(),
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
127
128
129
130
131
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
132
133
134
        # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
        if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
            model_type = hf_overrides.get("model_type", model_type)
135
136
137
138
139
140
141
142
143
144
145
146

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        else:
            try:
147
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
148
149
150
151
152
153
154
155
156
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    token=_get_hf_token(),
                    **kwargs,
                )
            except ValueError as e:
157
158
159
160
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
161
162
163
164
165
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
166
167
                        "`--trust-remote-code` flag in the CLI."
                    )
168
169
170
171
172
173
174
175
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
176
177
    def parse(
        self,
178
        model: str | Path,
179
        trust_remote_code: bool,
180
181
        revision: str | None = None,
        code_revision: str | None = None,
182
183
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
184
185
186
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
187
188
189
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
190
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
191
192
                model, revision, **kwargs
            )
193
194
195
196
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

197
198
199
200
201
202
203
204
205
206
207
208
209
        # Get missing fields from HF config if available
        try:
            hf_config_dict, _ = PretrainedConfig.get_config_dict(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        except OSError:  # Not found
            hf_config_dict = {}

        config = adapt_config_dict(config_dict, defaults=hf_config_dict)
210
211
212

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
213
214
215
        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
            sliding_window, list
        ):
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
264
265
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
266
267
268
269
270
271
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
272
273
274
275
276
277
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
278
279
280
281
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
282
        if not issubclass(config_parser_cls, ConfigParserBase):
283
284
285
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
286
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
287
288
289
290
291
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
292
293
294
        return config_parser_cls

    return _wrapper
295
296


297
298
299
300
301
302
303
304
305
306
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
    """Some models may have no rope_theta in their config but still use RoPE.
    This function sets a default rope_theta if it's missing."""
    if getattr(config, "rope_parameters", None) is None:
        config.rope_parameters = {"rope_type": "default"}
    if "rope_theta" not in config.rope_parameters:
        config.rope_parameters["rope_theta"] = default_theta


def patch_rope_parameters(config: PretrainedConfig) -> None:
307
    """Provide backwards compatibility for RoPE."""
308
309
    from vllm.config.utils import getattr_iter

310
311
312
313
314
315
316
    # Older custom models may use non-standard field names
    # which need patching for both Transformers v4 and v5.
    names = ["rope_theta", "rotary_emb_base"]
    rope_theta = getattr_iter(config, names, None, warn=True)
    names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"]
    partial_rotary_factor = getattr_iter(config, names, None, warn=True)

317
318
    if Version(version("transformers")) < Version("5.0.0.dev0"):
        # Transformers v4 installed, legacy config fields may be present
319
320
        if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
            config.rope_parameters = rope_scaling
321
        if rope_theta is not None:
322
323
324
            if not hasattr(config, "rope_parameters"):
                config.rope_parameters = {"rope_type": "default"}
            config.rope_parameters["rope_theta"] = rope_theta
325
326
327
328
        if partial_rotary_factor is not None:
            if not hasattr(config, "rope_parameters"):
                config.rope_parameters = {"rope_type": "default"}
            config.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
329
330
    elif rope_theta is not None or hasattr(config, "rope_parameters"):
        # Transformers v5 installed
331
332
333
334
335
336
        # Patch these fields in case they used non-standard names
        if rope_theta is not None:
            config.rope_theta = rope_theta
        if partial_rotary_factor is not None:
            config.partial_rotary_factor = partial_rotary_factor
        # Standardize and validate RoPE parameters
337
338
        config.standardize_rope_params()
        config.validate_rope()
339
340

    # No RoPE parameters to patch
341
    if getattr(config, "rope_parameters", None) is None:
342
343
        return

344
345
346
347
    # Add original_max_position_embeddings if present
    if ompe := getattr(config, "original_max_position_embeddings", None):
        config.rope_parameters["original_max_position_embeddings"] = ompe

348
    # Handle nested rope_parameters in interleaved sliding attention models
349
350
    if set(config.rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES):
        for rope_parameters_layer_type in config.rope_parameters.values():
351
352
            patch_rope_parameters_dict(rope_parameters_layer_type)
    else:
353
        patch_rope_parameters_dict(config.rope_parameters)
354
355


356
357
358
359
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
    if "rope_type" in rope_parameters and "type" in rope_parameters:
        rope_type = rope_parameters["rope_type"]
        rope_type_legacy = rope_parameters["type"]
360
361
362
363
364
        if (rope_type_legacy == "su" and rope_type == "longrope") or (
            rope_type_legacy == "mrope" and rope_type == "default"
        ):
            pass  # No action needed
        elif rope_type != rope_type_legacy:
365
366
367
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
368
369
                "You should only specify one of them."
            )
370

371
372
    if "rope_type" not in rope_parameters and "type" in rope_parameters:
        rope_parameters["rope_type"] = rope_parameters["type"]
373
374
        logger.info("Replacing legacy 'type' key with 'rope_type'")

375
376
    if "rope_type" not in rope_parameters:
        raise ValueError("rope_parameters should have a 'rope_type' key")
377

378
379
    if rope_parameters["rope_type"] == "su":
        rope_parameters["rope_type"] = "longrope"
380
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
381
    elif rope_parameters["rope_type"] == "mrope":
382
383
384
385
        if "mrope_section" not in rope_parameters:
            raise ValueError(
                "Legacy rope_type 'mrope' requires 'mrope_section' in rope_parameters"
            )
386
        rope_parameters["rope_type"] = "default"
387
388
389
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


390
def _uses_mrope(config: PretrainedConfig) -> bool:
391
392
    rope_parameters = getattr(config, "rope_parameters", None)
    if rope_parameters is None:
393
394
        return False

395
    return "mrope_section" in rope_parameters
396
397


398
399
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
400
401
402
403
404
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
def uses_xdrope_dim(config: PretrainedConfig) -> int:
    """Detect if the model with this config uses XD-ROPE."""
    xdrope_section = getattr(config, "xdrope_section", None)
    if xdrope_section is not None and isinstance(xdrope_section, list):
        return len(xdrope_section)
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return 0

    if isinstance(rope_scaling, dict) and "xdrope_section" in rope_scaling:
        xdrope_section = rope_scaling["xdrope_section"]
        if xdrope_section is not None and isinstance(xdrope_section, list):
            return len(xdrope_section)

    return 0


437
438
439
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

440
441
442
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

443
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
444
445


446
447
448
449
450
451
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
452
        return len(set(layer_types)) > 1
453
454
455
    return False


456
457
458
459
460
461
462
463
464
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


465
466
467
468
469
470
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
471
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
472
473
474
    return config


475
def maybe_override_with_speculators(
476
    model: str,
477
    tokenizer: str | None,
478
    trust_remote_code: bool,
479
480
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
481
    **kwargs,
482
) -> tuple[str, str | None, dict[str, Any] | None]:
483
    """
484
485
486
487
488
489
490
491
492
493
494
495
496
497
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
498
    """
499
    if check_gguf_file(model):
500
501
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
502
503
504
    elif is_remote_gguf(model):
        repo_id, _ = split_remote_gguf(model)
        gguf_model_repo = Path(repo_id)
505
506
    else:
        gguf_model_repo = None
507
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
508
    config_dict, _ = PretrainedConfig.get_config_dict(
509
        model if gguf_model_repo is None else gguf_model_repo,
510
511
512
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
513
        **kwargs,
514
    )
515
516
517
518
519
520
521
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
522
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
523

524
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
525
526
        config_dict=config_dict
    )
527
528

    # Set the draft model to the speculators model
529
    speculative_config["model"] = model
530
531
532
533
534

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

535
    return model, tokenizer, speculative_config
536
537


538
def get_config(
539
    model: str | Path,
540
    trust_remote_code: bool,
541
542
543
544
545
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
546
547
548
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
549

550
551
552
553
554
555
556
557
558
559
560
561
    _is_gguf = is_gguf(model)
    _is_remote_gguf = is_remote_gguf(model)
    if _is_gguf:
        if check_gguf_file(model):
            # Local GGUF file
            kwargs["gguf_file"] = Path(model).name
            model = Path(model).parent
        elif _is_remote_gguf:
            # Remote GGUF - extract repo_id from repo_id:quant_type format
            # The actual GGUF file will be downloaded later by GGUFModelLoader
            # Keep model as repo_id:quant_type for download, but use repo_id for config
            model, _ = split_remote_gguf(model)
562

563
    if config_format == "auto":
564
        try:
565
566
567
            # First check for Mistral to avoid defaulting to
            # Transformers implementation.
            if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
568
                config_format = "mistral"
569
            elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
570
571
572
                model, HF_CONFIG_NAME, revision=revision
            ):
                config_format = "hf"
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
            # Remote GGUF models must have config.json in repo,
            # otherwise the config can't be parsed correctly.
            # FIXME(Isotr0py): Support remote GGUF repos without config.json
            elif _is_remote_gguf and not file_or_path_exists(
                model, HF_CONFIG_NAME, revision=revision
            ):
                err_msg = (
                    "Could not find config.json for remote GGUF model repo. "
                    "To load remote GGUF model through `<repo_id>:<quant_type>`, "
                    "ensure your model has config.json (HF format) file. "
                    "Otherwise please specify --hf-config-path <original_repo> "
                    "in engine args to fetch config from unquantized hf model."
                )
                logger.error(err_msg)
                raise ValueError(err_msg)
588
589
590
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
591
592
593
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
594
595
                    "in engine args for customized config parser."
                )
596
597
598
599
600
601
602
603
604
605
606

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
607
                "'params.json'.\n"
608
            ).format(model=model)
609
610

            raise ValueError(error_message) from e
611

612
613
614
615
616
617
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
618
        hf_overrides=hf_overrides_kw,
619
620
        **kwargs,
    )
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642

    # Patching defaults for GGUF models
    if _is_gguf:
        # Some models have different default values between GGUF and HF.
        def apply_gguf_default(key: str, gguf_default: Any):
            """
            Apply GGUF defaults unless explicitly configured.

            This function reads/writes external `config` and `config_dict`.
            If the specified `key` is not in `config_dict` (i.e. not explicitly
            configured and the default HF value is used), it updates the
            corresponding `config` value to `gguf_default`.
            """
            if key not in config_dict:
                config.update({key: gguf_default})

        # Apply architecture-specific GGUF defaults.
        if config.model_type in {"qwen3_moe"}:
            # Qwen3 MoE: norm_topk_prob is always true.
            # Note that, this parameter is always false (HF default) on Qwen2 MoE.
            apply_gguf_default("norm_topk_prob", True)

643
    # Special architecture mapping check for GGUF models
644
    if _is_gguf:
645
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
646
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
647
648
649
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

650
651
652
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
653
654
655
656
657
658
659
660
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
661

662
663
664
665
666
667
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
668
669
670
671
672
673
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
674
675
676

    if quantization_config is not None:
        config.quantization_config = quantization_config
677
        # auto-enable DeepGEMM UE8M0 if model config requests it
678
        scale_fmt = quantization_config.get("scale_fmt", None)
679
        if scale_fmt in ("ue8m0",):
680
681
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
682
                logger.info_once(
683
684
                    (
                        "Detected quantization_config.scale_fmt=%s; "
685
                        "enabling UE8M0 for DeepGEMM."
686
                    ),
687
688
                    scale_fmt,
                )
689
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
690
                logger.warning_once(
691
692
693
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
694
695
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
696
                    ),
697
698
                    scale_fmt,
                )
699

700
701
702
703
704
705
706
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

707
708
709
710
711
712
713
714
    # Exhaustively patch RoPE parameters everywhere they might be
    patch_rope_parameters(config)
    patch_rope_parameters(config.get_text_config())
    SubConfigs: TypeAlias = dict[str, PretrainedConfig]
    sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
    if sub_configs:
        for sub_config in sub_configs:
            patch_rope_parameters(getattr(config, sub_config))
715

716
717
718
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

719
    return config
720
721


722
@cache
723
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None:
724
    """
725
726
727
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
728
729

    Args:
730
        model: The name of the Hugging Face model.
731
        revision: The specific version of the model to use.
732
            Defaults to 'main'.
733
734

    Returns:
735
        A dictionary containing the pooling type and whether
736
            normalization is used, or None if no pooling configuration is found.
737
    """
738
739
    if is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
740
741

    modules_file_name = "modules.json"
742
743

    modules_dict = None
744
745
746
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
747
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
748
749
750
751

    if modules_dict is None:
        return None

752
753
    logger.info("Found sentence-transformers modules configuration.")

754
755
756
757
758
759
760
761
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
762
    normalize = bool(
763
764
765
766
767
768
769
770
771
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
772
773
774

    if pooling:
        pooling_file_name = "{}/config.json".format(pooling["path"])
775
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
776
        pooling_type_name = next(
777
778
            (item for item, val in pooling_dict.items() if val is True), None
        )
779
780
781
782

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

783
        logger.info("Found pooling configuration.")
784
785
786
787
788
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


789
def get_pooling_config_name(pooling_name: str) -> str | None:
790
791
792
793
794
795
796
797
798
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

799
    supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"]
800
801
    pooling_type_name = pooling_name.upper()

802
803
804
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

805
    raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
806
807


808
@cache
809
def get_sentence_transformer_tokenizer_config(
810
    model: str | Path, revision: str | None = "main"
811
):
812
    """
813
    Returns the tokenization configuration dictionary for a
814
815
816
    given Sentence Transformer BERT model.

    Parameters:
817
    - model (str|Path): The name of the Sentence Transformer
818
819
820
821
822
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
823
    - dict: A dictionary containing the configuration parameters
824
825
    for the Sentence Transformer BERT model.
    """
826
827
828
829
830
831
832
833
834
835
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
836
837

    for config_file in sentence_transformer_config_files:
838
839
840
841
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
842
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
843
844
            if encoder_dict:
                break
845

846
    if not encoder_dict and not Path(model).is_absolute():
847
848
        try:
            # If model is on HuggingfaceHub, get the repo files
849
850
851
            repo_files = list_repo_files(
                model, revision=revision, token=_get_hf_token()
            )
852
        except Exception:
853
854
855
856
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
857
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
858
859
860
                if encoder_dict:
                    break

861
862
863
    if not encoder_dict:
        return None

864
865
    logger.info("Found sentence-transformers tokenize configuration.")

866
867
868
869
870
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


871
def maybe_register_config_serialize_by_value() -> None:
872
873
    """Try to register HF model configuration class to serialize by value

874
875
876
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
877

878
    Examples:
879

880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
904
905
    try:
        import transformers_modules
906

907
        transformers_modules_available = True
908
    except ImportError:
909
        transformers_modules_available = False
910
911
912
913
914

    try:
        import multiprocessing
        import pickle

915
916
        import cloudpickle

917
        from vllm.config import VllmConfig
918

919
920
921
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
922
        def _reduce_config(config: VllmConfig):
923
            return (pickle.loads, (cloudpickle.dumps(config),))
924

925
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
926

927
928
929
930
931
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
932
            from vllm.v1.executor.ray_utils import ray
933

934
935
936
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

937
938
939
940
941
942
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
943
944
            exc_info=e,
        )
945
946


947
def get_hf_image_processor_config(
948
949
950
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
951
    **kwargs,
952
) -> dict[str, Any]:
953
    # ModelScope does not provide an interface for image_processor
954
    if envs.VLLM_USE_MODELSCOPE:
955
        return dict()
956
    # Separate model folder from file path for GGUF models
957
    if check_gguf_file(model):
958
        model = Path(model).parent
959
960
    elif is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
961
962
963
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
964
965


966
967
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
968
    No op for pure text models.
969
    """
970
971
    text_config = config.get_text_config()

972
973
974
975
976
977
978
    if text_config is not config and not hasattr(text_config, "num_attention_heads"):
        raise ValueError(
            "The text_config extracted from the model config does not have "
            "`num_attention_heads` attribute. This indicates a mismatch "
            "between the model config and vLLM's expectations. Please "
            "ensure that the model config is compatible with vLLM."
        )
979
980

    return text_config
981
982
983
984
985


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
986
987
988
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
) -> GenerationConfig | None:
989
990
991
992
993
994
995
    # GGUF files don't have generation_config.json - their config is embedded
    # in the file header. Skip all filesystem lookups to avoid re-reading the
    # memory-mapped file, which can hang in multi-process scenarios when the
    # EngineCore process already has the file mapped.
    if is_gguf(model):
        return None

996
997
998
999
1000
1001
1002
1003
1004
1005
1006
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1007
                config_format=config_format,
1008
1009
1010
1011
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1012
1013


1014
1015
1016
def try_get_safetensors_metadata(
    model: str,
    *,
1017
    revision: str | None = None,
1018
1019
1020
1021
1022
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
1023
        token=_get_hf_token(),
1024
1025
1026
    )

    try:
1027
1028
1029
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1030
1031
    except Exception:
        return None
1032
1033
1034


def try_get_tokenizer_config(
1035
    pretrained_model_name_or_path: str | os.PathLike,
1036
    trust_remote_code: bool,
1037
1038
    revision: str | None = None,
) -> dict[str, Any] | None:
1039
1040
1041
1042
1043
1044
1045
1046
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1047
1048


1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

        dense_modules = [
            m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
        ]
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1083
1084
1085
def get_safetensors_params_metadata(
    model: str,
    *,
1086
    revision: str | None = None,
1087
1088
1089
1090
1091
1092
1093
1094
1095
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1096
1097
1098
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1111
1112
1113
1114
1115
1116
1117
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1118
1119
            f"and if the config file exists."
        )
1120
1121
1122
1123
1124
1125
1126
1127
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1128
1129
1130
1131
1132
1133
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1134
1135
1136
1137
1138
1139
1140
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1141
1142
            exc_info=e,
        )
1143
1144

    return max_position_embeddings