config.py 41.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
5
from collections.abc import Callable
6
from dataclasses import asdict
7
from functools import cache, partial
8
from importlib.metadata import version
9
from pathlib import Path
10
from typing import Any, Literal, TypeAlias
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
from huggingface_hub import get_safetensors_metadata
14
from packaging.version import Version
15
from transformers import GenerationConfig, PretrainedConfig
16
from transformers.models.auto.image_processing_auto import get_image_processor_config
17
18
19
20
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
21
from transformers.models.auto.tokenization_auto import get_tokenizer_config
22
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
23

24
from vllm import envs
25
from vllm.logger import init_logger
26
from vllm.transformers_utils.repo_utils import is_mistral_model_repo
27
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
28
29

from .config_parser_base import ConfigParserBase
30
31
32
33
34
35
from .gguf_utils import (
    check_gguf_file,
    is_gguf,
    is_remote_gguf,
    split_remote_gguf,
)
36
from .repo_utils import (
37
38
39
40
41
42
    file_or_path_exists,
    get_hf_file_to_dict,
    list_repo_files,
    try_get_local_file,
    with_retry,
)
43

44
45
46
47
48
49
50
51
52
try:
    # Transformers v5
    from transformers.configuration_utils import ALLOWED_ATTENTION_LAYER_TYPES
except ImportError:
    # Transformers v4
    from transformers.configuration_utils import (
        ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
    )

53
if envs.VLLM_USE_MODELSCOPE:
54
55
56
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
57

58
59
MISTRAL_CONFIG_NAME = "params.json"

60
61
logger = init_logger(__name__)

62

63
64
class LazyConfigDict(dict):
    def __getitem__(self, key):
65
66
67
        if isinstance(value := super().__getitem__(key), type):
            return value

68
        import vllm.transformers_utils.configs as configs
69

70
        return getattr(configs, value)
71
72
73


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
74
    afmoe="AfmoeConfig",
75
    bagel="BagelConfig",
76
    chatglm="ChatGLMConfig",
77
78
    colqwen3="ColQwen3Config",
    ops_colqwen3="OpsColQwen3Config",
79
    deepseek_vl_v2="DeepseekVLV2Config",
80
    deepseek_v32="DeepseekV3Config",
81
    flex_olmo="FlexOlmoConfig",
82
    funaudiochat="FunAudioChatConfig",
83
    hunyuan_vl="HunYuanVLConfig",
oscardev256's avatar
oscardev256 committed
84
    isaac="IsaacConfig",
85
    kimi_linear="KimiLinearConfig",
86
    kimi_vl="KimiVLConfig",
Roger Wang's avatar
Roger Wang committed
87
    kimi_k25="KimiK25Config",
88
89
90
91
92
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
93
    midashenglm="MiDashengLMConfig",
94
95
96
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
97
    olmo3="Olmo3Config",
98
99
100
101
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
csy0225's avatar
csy0225 committed
102
    step3p5="Step3p5Config",
Roger Wang's avatar
Roger Wang committed
103
    qwen3_asr="Qwen3ASRConfig",
104
    qwen3_next="Qwen3NextConfig",
105
106
    qwen3_5="Qwen3_5Config",
    qwen3_5_moe="Qwen3_5MoeConfig",
Paul Pak's avatar
Paul Pak committed
107
    lfm2_moe="Lfm2MoeConfig",
108
    tarsier2="Tarsier2Config",
109
)
110

111
112
113
114
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

115
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
116
    "internvl_chat": {"has_no_defaults_at_init": True},
117
    "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"},
118
    "NVLM_D": {"has_no_defaults_at_init": True},
119
120
}

121

122
123
124
125
126
127
128
129
def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool:
    """Check if rope_parameters is nested by layer types."""
    # Cannot be nested if rope_parameters is empty
    if not rope_parameters:
        return False
    return set(rope_parameters.keys()).issubset(ALLOWED_ATTENTION_LAYER_TYPES)


130
class HFConfigParser(ConfigParserBase):
131
132
    def parse(
        self,
133
        model: str | Path,
134
        trust_remote_code: bool,
135
136
        revision: str | None = None,
        code_revision: str | None = None,
137
138
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
139
140
141
142
143
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
144
            trust_remote_code=trust_remote_code,
145
146
147
148
149
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
150
151
152
153
154
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
155
156
157
        # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
        if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
            model_type = hf_overrides.get("model_type", model_type)
158
159
160
161
162
163
164

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
165
                trust_remote_code=trust_remote_code,
166
167
168
169
                **kwargs,
            )
        else:
            try:
170
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
171
172
173
174
175
176
177
178
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    **kwargs,
                )
            except ValueError as e:
179
180
181
182
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
183
184
185
186
187
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
188
189
                        "`--trust-remote-code` flag in the CLI."
                    )
190
191
192
193
194
195
196
197
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
198
199
    def parse(
        self,
200
        model: str | Path,
201
        trust_remote_code: bool,
202
203
        revision: str | None = None,
        code_revision: str | None = None,
204
205
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
206
207
208
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
209
210
211
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
212
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
213
214
                model, revision, **kwargs
            )
215
216
217
218
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

219
220
221
222
223
224
225
226
227
228
229
230
        # Get missing fields from HF config if available
        try:
            hf_config_dict, _ = PretrainedConfig.get_config_dict(
                model,
                revision=revision,
                code_revision=code_revision,
                **kwargs,
            )
        except OSError:  # Not found
            hf_config_dict = {}

        config = adapt_config_dict(config_dict, defaults=hf_config_dict)
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
272
273
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
274
275
276
277
278
279
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
280
281
282
283
284
285
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
286
287
288
289
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
290
        if not issubclass(config_parser_cls, ConfigParserBase):
291
292
293
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
294
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
295
296
297
298
299
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
300
301
302
        return config_parser_cls

    return _wrapper
303
304


305
306
307
308
309
310
311
312
313
314
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
    """Some models may have no rope_theta in their config but still use RoPE.
    This function sets a default rope_theta if it's missing."""
    if getattr(config, "rope_parameters", None) is None:
        config.rope_parameters = {"rope_type": "default"}
    if "rope_theta" not in config.rope_parameters:
        config.rope_parameters["rope_theta"] = default_theta


def patch_rope_parameters(config: PretrainedConfig) -> None:
315
    """Provide backwards compatibility for RoPE."""
316
317
    from vllm.config.utils import getattr_iter

318
319
320
321
322
323
    # Older custom models may use non-standard field names
    # which need patching for both Transformers v4 and v5.
    names = ["rope_theta", "rotary_emb_base"]
    rope_theta = getattr_iter(config, names, None, warn=True)
    names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"]
    partial_rotary_factor = getattr_iter(config, names, None, warn=True)
324
    ompe = getattr(config, "original_max_position_embeddings", None)
325

326
    if Version(version("transformers")) < Version("5.0.0"):
327
        # Transformers v4 installed, legacy config fields may be present
328
329
        if (rope_scaling := getattr(config, "rope_scaling", None)) is not None:
            config.rope_parameters = rope_scaling
330
        if (
331
332
333
            rope_theta is not None
            or partial_rotary_factor is not None
            or ompe is not None
334
335
        ) and not getattr(config, "rope_parameters", None):
            config.rope_parameters = {"rope_type": "default"}
336
        # Patch legacy fields into rope_parameters
337
        if rope_theta is not None:
338
            config.rope_parameters["rope_theta"] = rope_theta
339
340
        if partial_rotary_factor is not None:
            config.rope_parameters["partial_rotary_factor"] = partial_rotary_factor
341
342
        if ompe is not None:
            config.rope_parameters["original_max_position_embeddings"] = ompe
343
    elif rope_theta is not None or getattr(config, "rope_parameters", None):
344
        # Transformers v5 installed
345
346
347
348
349
350
        # Patch these fields in case they used non-standard names
        if rope_theta is not None:
            config.rope_theta = rope_theta
        if partial_rotary_factor is not None:
            config.partial_rotary_factor = partial_rotary_factor
        # Standardize and validate RoPE parameters
351
352
        config.standardize_rope_params()
        config.validate_rope()
353
354

    # No RoPE parameters to patch
355
    if getattr(config, "rope_parameters", None) is None:
356
357
358
        return

    # Handle nested rope_parameters in interleaved sliding attention models
359
    if is_rope_parameters_nested(config.rope_parameters):
360
        for rope_parameters_layer_type in config.rope_parameters.values():
361
362
            patch_rope_parameters_dict(rope_parameters_layer_type)
    else:
363
        patch_rope_parameters_dict(config.rope_parameters)
364
365


366
367
368
369
def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
    if "rope_type" in rope_parameters and "type" in rope_parameters:
        rope_type = rope_parameters["rope_type"]
        rope_type_legacy = rope_parameters["type"]
370
371
372
373
374
        if (rope_type_legacy == "su" and rope_type == "longrope") or (
            rope_type_legacy == "mrope" and rope_type == "default"
        ):
            pass  # No action needed
        elif rope_type != rope_type_legacy:
375
376
377
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
378
379
                "You should only specify one of them."
            )
380

381
382
    if "rope_type" not in rope_parameters and "type" in rope_parameters:
        rope_parameters["rope_type"] = rope_parameters["type"]
383
384
        logger.info("Replacing legacy 'type' key with 'rope_type'")

385
386
    if "rope_type" not in rope_parameters:
        raise ValueError("rope_parameters should have a 'rope_type' key")
387

388
389
    if rope_parameters["rope_type"] == "su":
        rope_parameters["rope_type"] = "longrope"
390
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
391
    elif rope_parameters["rope_type"] == "mrope":
392
393
394
395
        if "mrope_section" not in rope_parameters:
            raise ValueError(
                "Legacy rope_type 'mrope' requires 'mrope_section' in rope_parameters"
            )
396
        rope_parameters["rope_type"] = "default"
397
398
399
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


400
def _uses_mrope(config: PretrainedConfig) -> bool:
401
402
    rope_parameters = getattr(config, "rope_parameters", None)
    if rope_parameters is None:
403
404
        return False

405
    return "mrope_section" in rope_parameters
406
407


408
409
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
410
411
412
413
414
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
def uses_xdrope_dim(config: PretrainedConfig) -> int:
    """Detect if the model with this config uses XD-ROPE."""
    xdrope_section = getattr(config, "xdrope_section", None)
    if xdrope_section is not None and isinstance(xdrope_section, list):
        return len(xdrope_section)
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return 0

    if isinstance(rope_scaling, dict) and "xdrope_section" in rope_scaling:
        xdrope_section = rope_scaling["xdrope_section"]
        if xdrope_section is not None and isinstance(xdrope_section, list):
            return len(xdrope_section)

    return 0


447
448
449
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

450
451
452
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

453
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
454
455


456
457
458
459
460
461
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
462
        return len(set(layer_types)) > 1
463
464
465
    return False


466
467
468
469
470
471
472
473
474
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


475
476
477
478
479
480
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
481
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
482
483
484
    return config


485
def maybe_override_with_speculators(
486
    model: str,
487
    tokenizer: str | None,
488
    trust_remote_code: bool,
489
490
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
491
    **kwargs,
492
) -> tuple[str, str | None, dict[str, Any] | None]:
493
    """
494
495
496
497
498
499
500
501
502
503
504
505
506
507
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
508
    """
509
    if check_gguf_file(model):
510
511
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
512
513
514
    elif is_remote_gguf(model):
        repo_id, _ = split_remote_gguf(model)
        gguf_model_repo = Path(repo_id)
515
516
    else:
        gguf_model_repo = None
517
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
518
    config_dict, _ = PretrainedConfig.get_config_dict(
519
        model if gguf_model_repo is None else gguf_model_repo,
520
521
        revision=revision,
        trust_remote_code=trust_remote_code,
522
        **kwargs,
523
    )
524
525
526
527
528
529
530
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
531
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
532

533
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
534
535
        config_dict=config_dict
    )
536
537

    # Set the draft model to the speculators model
538
    speculative_config["model"] = model
539
540
541
542
543

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

544
    return model, tokenizer, speculative_config
545
546


547
def get_config(
548
    model: str | Path,
549
    trust_remote_code: bool,
550
551
552
553
554
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
555
556
557
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
558

559
560
561
562
563
564
565
566
567
568
569
570
    _is_gguf = is_gguf(model)
    _is_remote_gguf = is_remote_gguf(model)
    if _is_gguf:
        if check_gguf_file(model):
            # Local GGUF file
            kwargs["gguf_file"] = Path(model).name
            model = Path(model).parent
        elif _is_remote_gguf:
            # Remote GGUF - extract repo_id from repo_id:quant_type format
            # The actual GGUF file will be downloaded later by GGUFModelLoader
            # Keep model as repo_id:quant_type for download, but use repo_id for config
            model, _ = split_remote_gguf(model)
571

572
    if config_format == "auto":
573
        try:
574
575
            # First check for Mistral to avoid defaulting to
            # Transformers implementation.
576
577
578
579
580
            if is_mistral_model_repo(
                model_name_or_path=str(model), revision=revision
            ) and file_or_path_exists(
                model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
            ):
581
                config_format = "mistral"
582
            elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
583
584
585
                model, HF_CONFIG_NAME, revision=revision
            ):
                config_format = "hf"
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
            # Remote GGUF models must have config.json in repo,
            # otherwise the config can't be parsed correctly.
            # FIXME(Isotr0py): Support remote GGUF repos without config.json
            elif _is_remote_gguf and not file_or_path_exists(
                model, HF_CONFIG_NAME, revision=revision
            ):
                err_msg = (
                    "Could not find config.json for remote GGUF model repo. "
                    "To load remote GGUF model through `<repo_id>:<quant_type>`, "
                    "ensure your model has config.json (HF format) file. "
                    "Otherwise please specify --hf-config-path <original_repo> "
                    "in engine args to fetch config from unquantized hf model."
                )
                logger.error(err_msg)
                raise ValueError(err_msg)
601
602
603
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
604
605
606
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
607
608
                    "in engine args for customized config parser."
                )
609
610
611
612
613
614
615
616
617
618
619

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
620
                "'params.json'.\n"
621
            ).format(model=model)
622
623

            raise ValueError(error_message) from e
624

625
626
627
628
629
630
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
631
        hf_overrides=hf_overrides_kw,
632
633
        **kwargs,
    )
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655

    # Patching defaults for GGUF models
    if _is_gguf:
        # Some models have different default values between GGUF and HF.
        def apply_gguf_default(key: str, gguf_default: Any):
            """
            Apply GGUF defaults unless explicitly configured.

            This function reads/writes external `config` and `config_dict`.
            If the specified `key` is not in `config_dict` (i.e. not explicitly
            configured and the default HF value is used), it updates the
            corresponding `config` value to `gguf_default`.
            """
            if key not in config_dict:
                config.update({key: gguf_default})

        # Apply architecture-specific GGUF defaults.
        if config.model_type in {"qwen3_moe"}:
            # Qwen3 MoE: norm_topk_prob is always true.
            # Note that, this parameter is always false (HF default) on Qwen2 MoE.
            apply_gguf_default("norm_topk_prob", True)

656
    # Special architecture mapping check for GGUF models
657
    if _is_gguf:
658
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
659
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
660
661
662
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

663
664
665
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
666
667
668
669
670
671
672
673
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
674

675
676
677
678
679
680
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
681
682
683
684
685
686
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
687
688
689

    if quantization_config is not None:
        config.quantization_config = quantization_config
690
        # auto-enable DeepGEMM UE8M0 if model config requests it
691
        scale_fmt = quantization_config.get("scale_fmt", None)
692
        if scale_fmt in ("ue8m0",):
693
694
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
695
                logger.info_once(
696
697
                    (
                        "Detected quantization_config.scale_fmt=%s; "
698
                        "enabling UE8M0 for DeepGEMM."
699
                    ),
700
701
                    scale_fmt,
                )
702
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
703
                logger.warning_once(
704
705
706
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
707
708
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
709
                    ),
710
711
                    scale_fmt,
                )
712

713
714
715
716
717
718
719
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

720
721
722
723
724
725
726
727
    # Exhaustively patch RoPE parameters everywhere they might be
    patch_rope_parameters(config)
    patch_rope_parameters(config.get_text_config())
    SubConfigs: TypeAlias = dict[str, PretrainedConfig]
    sub_configs: SubConfigs | None = getattr(config, "sub_configs", None)
    if sub_configs:
        for sub_config in sub_configs:
            patch_rope_parameters(getattr(config, sub_config))
728

729
730
731
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

732
    return config
733
734


735
@cache
736
737
738
739
def get_pooling_config(
    model: str,
    revision: str | None = "main",
) -> dict[str, Any] | None:
740
    """
741
742
743
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
744
745

    Args:
746
        model: The name of the Hugging Face model.
747
        revision: The specific version of the model to use.
748
            Defaults to 'main'.
749
750

    Returns:
751
        A dictionary containing the pooling type and whether
752
            normalization is used, or None if no pooling configuration is found.
753
    """
754
755
    if is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
756
757

    modules_file_name = "modules.json"
758
759

    modules_dict = None
760
761
762
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
763
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
764
765
766
767

    if modules_dict is None:
        return None

768
769
    logger.info("Found sentence-transformers modules configuration.")

770
771
772
773
774
775
776
777
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
778
    normalize = bool(
779
780
781
782
783
784
785
786
787
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
788
789

    if pooling:
790
        from vllm.config.pooler import SEQ_POOLING_TYPES, TOK_POOLING_TYPES
791

792
793
        pooling_file_name = "{}/config.json".format(pooling["path"])
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision) or {}
794

795
        logger.info("Found pooling configuration.")
796

797
        config: dict[str, Any] = {"use_activation": normalize}
798
799
800
801
802
803
804
805
806
807
808
        for key, val in pooling_dict.items():
            if val is True:
                pooling_type = parse_pooling_type(key)
                if pooling_type in SEQ_POOLING_TYPES:
                    config["seq_pooling_type"] = pooling_type
                elif pooling_type in TOK_POOLING_TYPES:
                    config["tok_pooling_type"] = pooling_type
                else:
                    logger.debug("Skipping unrelated field: %r=%r", key, val)

        return config
809
810
811
812

    return None


813
def parse_pooling_type(pooling_name: str):
814
815
816
817
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
818
        pooling_name = pooling_name.split("_", 1)[0]
819
820
821
822

    if "lasttoken" in pooling_name:
        pooling_name = "last"

823
    return pooling_name.upper()
824
825


826
@cache
827
def get_sentence_transformer_tokenizer_config(
828
    model: str | Path, revision: str | None = "main"
829
) -> dict[str, Any] | None:
830
    """
831
    Returns the tokenization configuration dictionary for a
832
833
834
    given Sentence Transformer BERT model.

    Parameters:
835
    - model (str|Path): The name of the Sentence Transformer
836
837
838
839
840
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
841
    - dict: A dictionary containing the configuration parameters
842
843
    for the Sentence Transformer BERT model.
    """
844
845
846
847
848
849
850
851
852
853
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
854
855

    for config_file in sentence_transformer_config_files:
856
857
858
859
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
860
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
861
862
            if encoder_dict:
                break
863

864
    if not encoder_dict and not Path(model).is_absolute():
865
866
        try:
            # If model is on HuggingfaceHub, get the repo files
867
            repo_files = list_repo_files(model, revision=revision)
868
        except Exception:
869
870
871
872
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
873
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
874
875
876
                if encoder_dict:
                    break

877
878
879
    if not encoder_dict:
        return None

880
881
    logger.info("Found sentence-transformers tokenize configuration.")

882
883
884
885
886
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


887
def maybe_register_config_serialize_by_value() -> None:
888
889
    """Try to register HF model configuration class to serialize by value

890
891
892
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
893

894
    Examples:
895

896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
920
921
    try:
        import transformers_modules
922

923
        transformers_modules_available = True
924
    except ImportError:
925
        transformers_modules_available = False
926
927
928
929
930

    try:
        import multiprocessing
        import pickle

931
932
        import cloudpickle

933
        from vllm.config import VllmConfig
934

935
936
937
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
938
        def _reduce_config(config: VllmConfig):
939
            return (pickle.loads, (cloudpickle.dumps(config),))
940

941
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
942

943
944
945
946
947
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
948
            from vllm.v1.executor.ray_utils import ray
949

950
951
952
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

953
954
955
956
957
958
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
959
960
            exc_info=e,
        )
961
962


963
def get_hf_image_processor_config(
964
965
966
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
967
    **kwargs,
968
) -> dict[str, Any]:
969
    # ModelScope does not provide an interface for image_processor
970
    if envs.VLLM_USE_MODELSCOPE:
971
        return dict()
972
    # Separate model folder from file path for GGUF models
973
    if check_gguf_file(model):
974
        model = Path(model).parent
975
976
    elif is_remote_gguf(model):
        model, _ = split_remote_gguf(model)
977
978
979
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
980
981


982
983
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
984
    No op for pure text models.
985
    """
986
987
    text_config = config.get_text_config()

988
989
990
991
992
993
994
    if text_config is not config and not hasattr(text_config, "num_attention_heads"):
        raise ValueError(
            "The text_config extracted from the model config does not have "
            "`num_attention_heads` attribute. This indicates a mismatch "
            "between the model config and vLLM's expectations. Please "
            "ensure that the model config is compatible with vLLM."
        )
995
996

    return text_config
997
998
999
1000
1001


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
1002
1003
1004
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
) -> GenerationConfig | None:
1005
1006
1007
1008
1009
1010
1011
    # GGUF files don't have generation_config.json - their config is embedded
    # in the file header. Skip all filesystem lookups to avoid re-reading the
    # memory-mapped file, which can hang in multi-process scenarios when the
    # EngineCore process already has the file mapped.
    if is_gguf(model):
        return None

1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1023
                config_format=config_format,
1024
1025
1026
1027
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1028
1029


1030
1031
1032
def try_get_safetensors_metadata(
    model: str,
    *,
1033
    revision: str | None = None,
1034
1035
):
    get_safetensors_metadata_partial = partial(
1036
        get_safetensors_metadata, model, revision=revision
1037
1038
1039
    )

    try:
1040
1041
1042
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1043
1044
    except Exception:
        return None
1045
1046
1047


def try_get_tokenizer_config(
1048
    pretrained_model_name_or_path: str | os.PathLike,
1049
    trust_remote_code: bool,
1050
1051
    revision: str | None = None,
) -> dict[str, Any] | None:
1052
1053
1054
1055
1056
1057
1058
1059
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1060
1061


1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

1075
1076
1077
1078
1079
        _DENSE_MODULE_TYPES = {
            "sentence_transformers.models.Dense",
            "pylate.models.Dense.Dense",
        }
        dense_modules = [m for m in modules if m.get("type") in _DENSE_MODULE_TYPES]
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1098
1099
1100
def get_safetensors_params_metadata(
    model: str,
    *,
1101
    revision: str | None = None,
1102
1103
1104
1105
1106
1107
1108
1109
1110
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1111
1112
1113
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1126
1127
1128
1129
1130
1131
1132
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1133
1134
            f"and if the config file exists."
        )
1135
1136
1137
1138
1139
1140
1141
1142
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1143
1144
1145
1146
1147
1148
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1149
1150
1151
1152
1153
1154
1155
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1156
1157
            exc_info=e,
        )
1158
1159

    return max_position_embeddings