config.py 40 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import json
5
import os
6
import time
7
from collections.abc import Callable
8
from dataclasses import asdict
9
from functools import cache, partial
10
from pathlib import Path
11
from typing import Any, Literal, TypeVar
Jasmond L's avatar
Jasmond L committed
12

Joe Runde's avatar
Joe Runde committed
13
import huggingface_hub
14
15
16
17
18
from huggingface_hub import (
    get_safetensors_metadata,
    hf_hub_download,
    try_to_load_from_cache,
)
19
from huggingface_hub import list_repo_files as hf_list_repo_files
20
21
22
23
24
25
26
from huggingface_hub.utils import (
    EntryNotFoundError,
    HfHubHTTPError,
    LocalEntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
)
27
from transformers import GenerationConfig, PretrainedConfig
28
from transformers.models.auto.image_processing_auto import get_image_processor_config
29
30
31
32
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
33
from transformers.models.auto.tokenization_auto import get_tokenizer_config
34
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
35

36
from vllm import envs
37
from vllm.logger import init_logger
38
from vllm.transformers_utils.config_parser_base import ConfigParserBase
39
40
41
42
from vllm.transformers_utils.utils import (
    check_gguf_file,
    parse_safetensors_file_metadata,
)
43

44
if envs.VLLM_USE_MODELSCOPE:
45
46
47
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
48

49
50
MISTRAL_CONFIG_NAME = "params.json"

51
52
logger = init_logger(__name__)

53

54
def _get_hf_token() -> str | None:
55
56
57
    """
    Get the HuggingFace token from environment variable.

58
    Returns None if the token is not set, is an empty string,
59
60
61
62
    or contains only whitespace.
    This follows the same pattern as huggingface_hub library which
    treats empty string tokens as None to avoid authentication errors.
    """
63
    token = os.getenv("HF_TOKEN")
64
65
66
67
68
    if token and token.strip():
        return token
    return None


69
70
71
class LazyConfigDict(dict):
    def __getitem__(self, key):
        import vllm.transformers_utils.configs as configs
72

73
74
75
76
77
78
        return getattr(configs, super().__getitem__(key))


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
    chatglm="ChatGLMConfig",
    deepseek_vl_v2="DeepseekVLV2Config",
79
80
    deepseek_v3="DeepseekV3Config",
    deepseek_v32="DeepseekV3Config",
81
    flex_olmo="FlexOlmoConfig",
82
    kimi_linear="KimiLinearConfig",
83
84
85
86
87
88
89
    kimi_vl="KimiVLConfig",
    Llama_Nemotron_Nano_VL="Nemotron_Nano_VL_Config",
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
90
    midashenglm="MiDashengLMConfig",
91
92
93
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
94
    olmo3="Olmo3Config",
95
96
97
98
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
99
    qwen3_next="Qwen3NextConfig",
Paul Pak's avatar
Paul Pak committed
100
    lfm2_moe="Lfm2MoeConfig",
101
)
102

103
104
105
106
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

107
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
108
109
    "internvl_chat": {"has_no_defaults_at_init": True},
    "NVLM_D": {"has_no_defaults_at_init": True},
110
111
}

112

113
class HFConfigParser(ConfigParserBase):
114
115
    def parse(
        self,
116
        model: str | Path,
117
        trust_remote_code: bool,
118
119
        revision: str | None = None,
        code_revision: str | None = None,
120
121
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
122
123
124
125
126
127
128
129
130
131
132
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            token=_get_hf_token(),
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
133
134
135
136
137
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
138
139
140
141
142
143
144
145
146
147
148
149

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        else:
            try:
150
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
151
152
153
154
155
156
157
158
159
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    token=_get_hf_token(),
                    **kwargs,
                )
            except ValueError as e:
160
161
162
163
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
164
165
166
167
168
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
169
170
                        "`--trust-remote-code` flag in the CLI."
                    )
171
172
173
174
175
176
177
178
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
179
180
    def parse(
        self,
181
        model: str | Path,
182
        trust_remote_code: bool,
183
184
        revision: str | None = None,
        code_revision: str | None = None,
185
186
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
187
188
189
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
190
191
192
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
193
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
194
195
                model, revision, **kwargs
            )
196
197
198
199
200
201
202
203
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

        config = adapt_config_dict(config_dict)

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
204
205
206
        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
            sliding_window, list
        ):
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
255
256
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
257
258
259
260
261
262
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
263
264
265
266
267
268
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
269
270
271
272
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
273
        if not issubclass(config_parser_cls, ConfigParserBase):
274
275
276
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
277
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
278
279
280
281
282
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
283
284
285
        return config_parser_cls

    return _wrapper
286
287


288
289
290
291
292
293
294
295
296
_R = TypeVar("_R")


def with_retry(
    func: Callable[[], _R],
    log_msg: str,
    max_retries: int = 2,
    retry_delay: int = 2,
) -> _R:
297
298
299
300
301
302
303
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                logger.error("%s: %s", log_msg, e)
                raise
304
305
306
            logger.error(
                "%s: %s, retrying %d of %d", log_msg, e, attempt + 1, max_retries
            )
307
308
309
            time.sleep(retry_delay)
            retry_delay *= 2

310
311
    raise AssertionError("Should not be reached")

312
313
314
315
316
317

# @cache doesn't cache exceptions
@cache
def list_repo_files(
    repo_id: str,
    *,
318
319
320
    revision: str | None = None,
    repo_type: str | None = None,
    token: str | bool | None = None,
321
) -> list[str]:
322
323
324
325
326
    def lookup_files() -> list[str]:
        # directly list files if model is local
        if (local_path := Path(repo_id)).exists():
            return [
                str(file.relative_to(local_path))
327
328
                for file in local_path.rglob("*")
                if file.is_file()
329
330
            ]
        # if model is remote, use hf_hub api to list files
331
        try:
332
            if envs.VLLM_USE_MODELSCOPE:
333
334
335
336
337
338
339
340
341
342
                from vllm.transformers_utils.utils import modelscope_list_repo_files

                return modelscope_list_repo_files(
                    repo_id,
                    revision=revision,
                    token=os.getenv("MODELSCOPE_API_TOKEN", None),
                )
            return hf_list_repo_files(
                repo_id, revision=revision, repo_type=repo_type, token=token
            )
343
344
345
346
347
348
349
350
351
352
353
354
355
        except huggingface_hub.errors.OfflineModeIsEnabled:
            # Don't raise in offline mode,
            # all we know is that we don't have this
            # file cached.
            return []

    return with_retry(lookup_files, "Error retrieving file list")


def file_exists(
    repo_id: str,
    file_name: str,
    *,
356
357
358
    repo_type: str | None = None,
    revision: str | None = None,
    token: str | bool | None = None,
359
) -> bool:
360
361
362
    file_list = list_repo_files(
        repo_id, repo_type=repo_type, revision=revision, token=token
    )
363
364
365
366
    return file_name in file_list


# In offline mode the result can be a false negative
367
def file_or_path_exists(
368
    model: str | Path, config_name: str, revision: str | None
369
) -> bool:
370
371
    if (local_path := Path(model)).exists():
        return (local_path / config_name).is_file()
372

Joe Runde's avatar
Joe Runde committed
373
    # Offline mode support: Check if config file is cached already
374
375
376
    cached_filepath = try_to_load_from_cache(
        repo_id=model, filename=config_name, revision=revision
    )
Joe Runde's avatar
Joe Runde committed
377
378
379
380
381
382
    if isinstance(cached_filepath, str):
        # The config file exists in cache- we can continue trying to load
        return True

    # NB: file_exists will only check for the existence of the config file on
    # hf_hub. This will fail in offline mode.
383
384

    # Call HF to check if the file exists
385
386
387
    return file_exists(
        str(model), config_name, revision=revision, token=_get_hf_token()
    )
388
389


390
391
392
393
394
395
396
397
398
399
400
def patch_rope_scaling(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        patch_rope_scaling(text_config)

    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is not None:
        patch_rope_scaling_dict(rope_scaling)


401
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
402
403
404
405
406
407
408
    if "rope_type" in rope_scaling and "type" in rope_scaling:
        rope_type = rope_scaling["rope_type"]
        rope_type_legacy = rope_scaling["type"]
        if rope_type != rope_type_legacy:
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
409
410
                "You should only specify one of them."
            )
411

412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
    if "rope_type" not in rope_scaling and "type" in rope_scaling:
        rope_scaling["rope_type"] = rope_scaling["type"]
        logger.info("Replacing legacy 'type' key with 'rope_type'")

    if "rope_type" not in rope_scaling:
        raise ValueError("rope_scaling should have a 'rope_type' key")

    if rope_scaling["rope_type"] == "su":
        rope_scaling["rope_type"] = "longrope"
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
    elif rope_scaling["rope_type"] == "mrope":
        assert "mrope_section" in rope_scaling
        rope_scaling["rope_type"] = "default"
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


428
def _uses_mrope(config: PretrainedConfig) -> bool:
429
430
431
432
433
434
435
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return False

    return "mrope_section" in rope_scaling


436
437
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
438
439
440
441
442
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


458
459
460
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

461
462
463
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

464
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
465
466


467
468
469
470
471
472
473
474
475
476
477
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
        interleaved_types = {"full_attention", "sliding_attention"}
        return interleaved_types.issubset(layer_types)
    return False


478
479
480
481
482
483
484
485
486
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


487
488
489
490
491
492
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
493
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
494
495
496
    return config


497
def maybe_override_with_speculators(
498
499
500
    model: str,
    tokenizer: str,
    trust_remote_code: bool,
501
502
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
503
    **kwargs,
504
) -> tuple[str, str, dict[str, Any] | None]:
505
    """
506
507
508
509
510
511
512
513
514
515
516
517
518
519
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
520
    """
521
522
523
524
525
526
    is_gguf = check_gguf_file(model)
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
    else:
        gguf_model_repo = None
527
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
528
    config_dict, _ = PretrainedConfig.get_config_dict(
529
        model if gguf_model_repo is None else gguf_model_repo,
530
531
532
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
533
        **kwargs,
534
    )
535
536
537
538
539
540
541
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
542
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
543

544
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
545
546
        config_dict=config_dict
    )
547
548

    # Set the draft model to the speculators model
549
    speculative_config["model"] = model
550
551
552
553
554

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

555
    return model, tokenizer, speculative_config
556
557


558
def get_config(
559
    model: str | Path,
560
    trust_remote_code: bool,
561
562
563
564
565
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
566
567
568
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
569

570
    is_gguf = check_gguf_file(model)
571
572
573
574
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        model = Path(model).parent

575
    if config_format == "auto":
576
        try:
577
            if is_gguf or file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
578
                config_format = "hf"
579
            elif file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
580
                config_format = "mistral"
581
582
583
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
584
585
586
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
587
588
                    "in engine args for customized config parser."
                )
589
590
591
592
593
594
595
596
597
598
599

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
600
601
602
                "'params.json'.\n"
                "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
                "   Loading GGUF from a remote repo directly is not yet "
603
604
                "supported.\n"
            ).format(model=model)
605
606

            raise ValueError(error_message) from e
607

608
609
610
611
612
613
614
615
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
        **kwargs,
    )
616
617
618
    # Special architecture mapping check for GGUF models
    if is_gguf:
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
619
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
620
621
622
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

623
624
625
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
626
627
628
629
630
631
632
633
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
634

635
636
637
638
639
640
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
641
642
643
644
645
646
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
647
648
649

    if quantization_config is not None:
        config.quantization_config = quantization_config
650
        # auto-enable DeepGEMM UE8M0 if model config requests it
651
        scale_fmt = quantization_config.get("scale_fmt", None)
652
        if scale_fmt in ("ue8m0",):
653
654
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
655
                logger.info_once(
656
657
                    (
                        "Detected quantization_config.scale_fmt=%s; "
658
                        "enabling UE8M0 for DeepGEMM."
659
                    ),
660
661
                    scale_fmt,
                )
662
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
663
                logger.warning_once(
664
665
666
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
667
668
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
669
                    ),
670
671
                    scale_fmt,
                )
672

673
674
675
676
677
678
679
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

680
681
    patch_rope_scaling(config)

682
683
684
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

685
    return config
686
687


688
def try_get_local_file(
689
690
    model: str | Path, file_name: str, revision: str | None = "main"
) -> Path | None:
691
692
693
694
695
    file_path = Path(model) / file_name
    if file_path.is_file():
        return file_path
    else:
        try:
696
697
698
            cached_filepath = try_to_load_from_cache(
                repo_id=model, filename=file_name, revision=revision
            )
699
700
            if isinstance(cached_filepath, str):
                return Path(cached_filepath)
701
        except ValueError:
702
703
704
705
            ...
    return None


706
def get_hf_file_to_dict(
707
    file_name: str, model: str | Path, revision: str | None = "main"
708
):
709
    """
710
    Downloads a file from the Hugging Face Hub and returns
711
712
713
714
715
    its contents as a dictionary.

    Parameters:
    - file_name (str): The name of the file to download.
    - model (str): The name of the model on the Hugging Face Hub.
716
    - revision (str): The specific version of the model.
717
718

    Returns:
719
    - config_dict (dict): A dictionary containing
720
721
722
    the contents of the downloaded file.
    """

723
    file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
724

725
    if file_path is None:
726
727
        try:
            hf_hub_file = hf_hub_download(model, file_name, revision=revision)
728
729
        except huggingface_hub.errors.OfflineModeIsEnabled:
            return None
730
731
732
733
734
735
        except (
            RepositoryNotFoundError,
            RevisionNotFoundError,
            EntryNotFoundError,
            LocalEntryNotFoundError,
        ) as e:
736
737
738
739
            logger.debug("File or repository not found in hf_hub_download", e)
            return None
        except HfHubHTTPError as e:
            logger.warning(
740
                "Cannot connect to Hugging Face Hub. Skipping file download for '%s':",
741
                file_name,
742
743
                exc_info=e,
            )
744
745
746
747
            return None
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
748
749
        with open(file_path) as file:
            return json.load(file)
750

751
752
753
    return None


754
@cache
755
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None:
756
    """
757
758
759
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
760
761

    Args:
762
        model: The name of the Hugging Face model.
763
        revision: The specific version of the model to use.
764
            Defaults to 'main'.
765
766

    Returns:
767
        A dictionary containing the pooling type and whether
768
            normalization is used, or None if no pooling configuration is found.
769
770
771
    """

    modules_file_name = "modules.json"
772
773

    modules_dict = None
774
775
776
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
777
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
778
779
780
781

    if modules_dict is None:
        return None

782
783
    logger.info("Found sentence-transformers modules configuration.")

784
785
786
787
788
789
790
791
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
792
    normalize = bool(
793
794
795
796
797
798
799
800
801
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
802
803
804

    if pooling:
        pooling_file_name = "{}/config.json".format(pooling["path"])
805
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
806
        pooling_type_name = next(
807
808
            (item for item, val in pooling_dict.items() if val is True), None
        )
809
810
811
812

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

813
        logger.info("Found pooling configuration.")
814
815
816
817
818
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


819
def get_pooling_config_name(pooling_name: str) -> str | None:
820
821
822
823
824
825
826
827
828
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

829
    supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"]
830
831
    pooling_type_name = pooling_name.upper()

832
833
834
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

835
    raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
836
837


838
@cache
839
def get_sentence_transformer_tokenizer_config(
840
    model: str | Path, revision: str | None = "main"
841
):
842
    """
843
    Returns the tokenization configuration dictionary for a
844
845
846
    given Sentence Transformer BERT model.

    Parameters:
847
    - model (str|Path): The name of the Sentence Transformer
848
849
850
851
852
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
853
    - dict: A dictionary containing the configuration parameters
854
855
    for the Sentence Transformer BERT model.
    """
856
857
858
859
860
861
862
863
864
865
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
866
867

    for config_file in sentence_transformer_config_files:
868
869
870
871
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
872
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
873
874
            if encoder_dict:
                break
875

876
    if not encoder_dict and not Path(model).is_absolute():
877
878
        try:
            # If model is on HuggingfaceHub, get the repo files
879
880
881
            repo_files = list_repo_files(
                model, revision=revision, token=_get_hf_token()
            )
882
        except Exception:
883
884
885
886
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
887
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
888
889
890
                if encoder_dict:
                    break

891
892
893
    if not encoder_dict:
        return None

894
895
    logger.info("Found sentence-transformers tokenize configuration.")

896
897
898
899
900
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


901
def maybe_register_config_serialize_by_value() -> None:
902
903
    """Try to register HF model configuration class to serialize by value

904
905
906
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
907

908
    Examples:
909

910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
934
935
    try:
        import transformers_modules
936

937
        transformers_modules_available = True
938
    except ImportError:
939
        transformers_modules_available = False
940
941
942
943
944

    try:
        import multiprocessing
        import pickle

945
946
        import cloudpickle

947
        from vllm.config import VllmConfig
948

949
950
951
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
952
        def _reduce_config(config: VllmConfig):
953
            return (pickle.loads, (cloudpickle.dumps(config),))
954

955
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
956

957
958
959
960
961
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
962
            from vllm.v1.executor.ray_utils import ray
963

964
965
966
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

967
968
969
970
971
972
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
973
974
            exc_info=e,
        )
975
976


977
def get_hf_image_processor_config(
978
979
980
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
981
    **kwargs,
982
) -> dict[str, Any]:
983
    # ModelScope does not provide an interface for image_processor
984
    if envs.VLLM_USE_MODELSCOPE:
985
        return dict()
986
    # Separate model folder from file path for GGUF models
987
    if check_gguf_file(model):
988
        model = Path(model).parent
989
990
991
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
992
993


994
995
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
996
    No op for pure text models.
997
    """
998
999
1000
1001
1002
1003
1004
1005
1006
    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
1007
1008
1009
1010
1011


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
1012
1013
1014
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
) -> GenerationConfig | None:
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1026
                config_format=config_format,
1027
1028
1029
1030
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1031
1032


1033
1034
1035
def try_get_safetensors_metadata(
    model: str,
    *,
1036
    revision: str | None = None,
1037
1038
1039
1040
1041
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
1042
        token=_get_hf_token(),
1043
1044
1045
    )

    try:
1046
1047
1048
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1049
1050
    except Exception:
        return None
1051
1052
1053


def try_get_tokenizer_config(
1054
    pretrained_model_name_or_path: str | os.PathLike,
1055
    trust_remote_code: bool,
1056
1057
    revision: str | None = None,
) -> dict[str, Any] | None:
1058
1059
1060
1061
1062
1063
1064
1065
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1066
1067


1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

        dense_modules = [
            m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
        ]
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1102
1103
1104
def get_safetensors_params_metadata(
    model: str,
    *,
1105
    revision: str | None = None,
1106
1107
1108
1109
1110
1111
1112
1113
1114
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1115
1116
1117
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1130
1131
1132
1133
1134
1135
1136
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1137
1138
            f"and if the config file exists."
        )
1139
1140
1141
1142
1143
1144
1145
1146
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1147
1148
1149
1150
1151
1152
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1153
1154
1155
1156
1157
1158
1159
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1160
1161
            exc_info=e,
        )
1162
1163

    return max_position_embeddings
1164
1165


1166
def get_model_path(model: str | Path, revision: str | None = None):
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
    if os.path.exists(model):
        return model
    assert huggingface_hub.constants.HF_HUB_OFFLINE
    common_kwargs = {
        "local_files_only": huggingface_hub.constants.HF_HUB_OFFLINE,
        "revision": revision,
    }

    if envs.VLLM_USE_MODELSCOPE:
        from modelscope.hub.snapshot_download import snapshot_download
1177

1178
1179
1180
        return snapshot_download(model_id=model, **common_kwargs)

    from huggingface_hub import snapshot_download
1181

1182
    return snapshot_download(repo_id=model, **common_kwargs)
1183
1184


1185
def get_hf_file_bytes(
1186
1187
    file_name: str, model: str | Path, revision: str | None = "main"
) -> bytes | None:
1188
    """Get file contents from HuggingFace repository as bytes."""
1189
    file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
1190
1191

    if file_path is None:
1192
1193
1194
        hf_hub_file = hf_hub_download(
            model, file_name, revision=revision, token=_get_hf_token()
        )
1195
1196
1197
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
1198
        with open(file_path, "rb") as file:
1199
1200
1201
            return file.read()

    return None