config.py 40.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import json
5
import os
6
import time
7
from collections.abc import Callable
8
from dataclasses import asdict
9
from functools import cache, partial
10
from pathlib import Path
11
from typing import Any, Literal, TypeVar
Jasmond L's avatar
Jasmond L committed
12

Joe Runde's avatar
Joe Runde committed
13
import huggingface_hub
14
15
16
17
18
from huggingface_hub import (
    get_safetensors_metadata,
    hf_hub_download,
    try_to_load_from_cache,
)
19
from huggingface_hub import list_repo_files as hf_list_repo_files
20
21
22
23
24
25
26
from huggingface_hub.utils import (
    EntryNotFoundError,
    HfHubHTTPError,
    LocalEntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
)
27
from transformers import DeepseekV3Config, GenerationConfig, PretrainedConfig
28
from transformers.models.auto.image_processing_auto import get_image_processor_config
29
30
31
32
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    MODEL_MAPPING_NAMES,
)
33
from transformers.models.auto.tokenization_auto import get_tokenizer_config
34
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
35

36
from vllm import envs
37
from vllm.logger import init_logger
38
from vllm.transformers_utils.config_parser_base import ConfigParserBase
39
40
41
42
from vllm.transformers_utils.utils import (
    check_gguf_file,
    parse_safetensors_file_metadata,
)
43

44
if envs.VLLM_USE_MODELSCOPE:
45
46
47
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
48

49
50
MISTRAL_CONFIG_NAME = "params.json"

51
52
logger = init_logger(__name__)

53

54
def _get_hf_token() -> str | None:
55
56
57
    """
    Get the HuggingFace token from environment variable.

58
    Returns None if the token is not set, is an empty string,
59
60
61
62
    or contains only whitespace.
    This follows the same pattern as huggingface_hub library which
    treats empty string tokens as None to avoid authentication errors.
    """
63
    token = os.getenv("HF_TOKEN")
64
65
66
67
68
    if token and token.strip():
        return token
    return None


69
70
class LazyConfigDict(dict):
    def __getitem__(self, key):
71
72
73
        if isinstance(value := super().__getitem__(key), type):
            return value

74
        import vllm.transformers_utils.configs as configs
75

76
        return getattr(configs, value)
77
78
79


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
80
    afmoe="AfmoeConfig",
81
82
    chatglm="ChatGLMConfig",
    deepseek_vl_v2="DeepseekVLV2Config",
83
    deepseek_v32=DeepseekV3Config,
84
    flex_olmo="FlexOlmoConfig",
85
    kimi_linear="KimiLinearConfig",
86
87
88
89
90
91
    kimi_vl="KimiVLConfig",
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
92
    midashenglm="MiDashengLMConfig",
93
94
95
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
96
    olmo3="Olmo3Config",
97
98
99
100
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
101
    qwen3_next="Qwen3NextConfig",
Paul Pak's avatar
Paul Pak committed
102
    lfm2_moe="Lfm2MoeConfig",
103
)
104

105
106
107
108
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

109
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
110
    "internvl_chat": {"has_no_defaults_at_init": True},
111
    "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"},
112
    "NVLM_D": {"has_no_defaults_at_init": True},
113
114
}

115

116
class HFConfigParser(ConfigParserBase):
117
118
    def parse(
        self,
119
        model: str | Path,
120
        trust_remote_code: bool,
121
122
        revision: str | None = None,
        code_revision: str | None = None,
123
124
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
125
126
127
128
129
130
131
132
133
134
135
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            token=_get_hf_token(),
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
136
137
138
139
140
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
141
142
143
144
145
146
147
148
149
150
151
152

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        else:
            try:
153
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
154
155
156
157
158
159
160
161
162
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    token=_get_hf_token(),
                    **kwargs,
                )
            except ValueError as e:
163
164
165
166
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
167
168
169
170
171
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
172
173
                        "`--trust-remote-code` flag in the CLI."
                    )
174
175
176
177
178
179
180
181
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
182
183
    def parse(
        self,
184
        model: str | Path,
185
        trust_remote_code: bool,
186
187
        revision: str | None = None,
        code_revision: str | None = None,
188
189
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
190
191
192
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
193
194
195
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
196
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
197
198
                model, revision, **kwargs
            )
199
200
201
202
203
204
205
206
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

        config = adapt_config_dict(config_dict)

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
207
208
209
        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
            sliding_window, list
        ):
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
258
259
         ...         revision: str | None = None,
         ...         code_revision: str | None = None,
260
261
262
263
264
265
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
266
267
268
269
270
271
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
272
273
274
275
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
276
        if not issubclass(config_parser_cls, ConfigParserBase):
277
278
279
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
280
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
281
282
283
284
285
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
286
287
288
        return config_parser_cls

    return _wrapper
289
290


291
292
293
294
295
296
297
298
299
_R = TypeVar("_R")


def with_retry(
    func: Callable[[], _R],
    log_msg: str,
    max_retries: int = 2,
    retry_delay: int = 2,
) -> _R:
300
301
302
303
304
305
306
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                logger.error("%s: %s", log_msg, e)
                raise
307
308
309
            logger.error(
                "%s: %s, retrying %d of %d", log_msg, e, attempt + 1, max_retries
            )
310
311
312
            time.sleep(retry_delay)
            retry_delay *= 2

313
314
    raise AssertionError("Should not be reached")

315
316
317
318
319
320

# @cache doesn't cache exceptions
@cache
def list_repo_files(
    repo_id: str,
    *,
321
322
323
    revision: str | None = None,
    repo_type: str | None = None,
    token: str | bool | None = None,
324
) -> list[str]:
325
326
327
328
329
    def lookup_files() -> list[str]:
        # directly list files if model is local
        if (local_path := Path(repo_id)).exists():
            return [
                str(file.relative_to(local_path))
330
331
                for file in local_path.rglob("*")
                if file.is_file()
332
333
            ]
        # if model is remote, use hf_hub api to list files
334
        try:
335
            if envs.VLLM_USE_MODELSCOPE:
336
337
338
339
340
341
342
343
344
345
                from vllm.transformers_utils.utils import modelscope_list_repo_files

                return modelscope_list_repo_files(
                    repo_id,
                    revision=revision,
                    token=os.getenv("MODELSCOPE_API_TOKEN", None),
                )
            return hf_list_repo_files(
                repo_id, revision=revision, repo_type=repo_type, token=token
            )
346
347
348
349
350
351
352
353
354
355
356
357
358
        except huggingface_hub.errors.OfflineModeIsEnabled:
            # Don't raise in offline mode,
            # all we know is that we don't have this
            # file cached.
            return []

    return with_retry(lookup_files, "Error retrieving file list")


def file_exists(
    repo_id: str,
    file_name: str,
    *,
359
360
361
    repo_type: str | None = None,
    revision: str | None = None,
    token: str | bool | None = None,
362
) -> bool:
363
364
365
    file_list = list_repo_files(
        repo_id, repo_type=repo_type, revision=revision, token=token
    )
366
367
368
369
    return file_name in file_list


# In offline mode the result can be a false negative
370
def file_or_path_exists(
371
    model: str | Path, config_name: str, revision: str | None
372
) -> bool:
373
374
    if (local_path := Path(model)).exists():
        return (local_path / config_name).is_file()
375

Joe Runde's avatar
Joe Runde committed
376
    # Offline mode support: Check if config file is cached already
377
378
379
    cached_filepath = try_to_load_from_cache(
        repo_id=model, filename=config_name, revision=revision
    )
Joe Runde's avatar
Joe Runde committed
380
381
382
383
384
385
    if isinstance(cached_filepath, str):
        # The config file exists in cache- we can continue trying to load
        return True

    # NB: file_exists will only check for the existence of the config file on
    # hf_hub. This will fail in offline mode.
386
387

    # Call HF to check if the file exists
388
389
390
    return file_exists(
        str(model), config_name, revision=revision, token=_get_hf_token()
    )
391
392


393
394
395
396
397
398
399
400
401
402
403
def patch_rope_scaling(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        patch_rope_scaling(text_config)

    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is not None:
        patch_rope_scaling_dict(rope_scaling)


404
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
405
406
407
408
409
410
411
    if "rope_type" in rope_scaling and "type" in rope_scaling:
        rope_type = rope_scaling["rope_type"]
        rope_type_legacy = rope_scaling["type"]
        if rope_type != rope_type_legacy:
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
412
413
                "You should only specify one of them."
            )
414

415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
    if "rope_type" not in rope_scaling and "type" in rope_scaling:
        rope_scaling["rope_type"] = rope_scaling["type"]
        logger.info("Replacing legacy 'type' key with 'rope_type'")

    if "rope_type" not in rope_scaling:
        raise ValueError("rope_scaling should have a 'rope_type' key")

    if rope_scaling["rope_type"] == "su":
        rope_scaling["rope_type"] = "longrope"
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
    elif rope_scaling["rope_type"] == "mrope":
        assert "mrope_section" in rope_scaling
        rope_scaling["rope_type"] = "default"
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


431
def _uses_mrope(config: PretrainedConfig) -> bool:
432
433
434
435
436
437
438
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return False

    return "mrope_section" in rope_scaling


439
440
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
441
442
443
444
445
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


461
462
463
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

464
465
466
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

467
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
468
469


470
471
472
473
474
475
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
476
        return len(set(layer_types)) > 1
477
478
479
    return False


480
481
482
483
484
485
486
487
488
489
490
def uses_custom_attention_masks(config: PretrainedConfig) -> bool:
    """Detect if model uses custom attention mask generation for multimodal.

    Some multimodal models require custom attention masks that enable
    bidirectional attention between image tokens while maintaining causal
    attention for text tokens. Currently applies to Gemma3 multimodal models.
    """
    architectures = getattr(config, "architectures", [])
    return "Gemma3ForConditionalGeneration" in architectures


491
492
493
494
495
496
497
498
499
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


500
501
502
503
504
505
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
506
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
507
508
509
    return config


510
def maybe_override_with_speculators(
511
512
513
    model: str,
    tokenizer: str,
    trust_remote_code: bool,
514
515
    revision: str | None = None,
    vllm_speculative_config: dict[str, Any] | None = None,
516
    **kwargs,
517
) -> tuple[str, str, dict[str, Any] | None]:
518
    """
519
520
521
522
523
524
525
526
527
528
529
530
531
532
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
533
    """
534
535
536
537
538
539
    is_gguf = check_gguf_file(model)
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
    else:
        gguf_model_repo = None
540
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
541
    config_dict, _ = PretrainedConfig.get_config_dict(
542
        model if gguf_model_repo is None else gguf_model_repo,
543
544
545
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
546
        **kwargs,
547
    )
548
549
550
551
552
553
554
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
555
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
556

557
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
558
559
        config_dict=config_dict
    )
560
561

    # Set the draft model to the speculators model
562
    speculative_config["model"] = model
563
564
565
566
567

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

568
    return model, tokenizer, speculative_config
569
570


571
def get_config(
572
    model: str | Path,
573
    trust_remote_code: bool,
574
575
576
577
578
    revision: str | None = None,
    code_revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
    hf_overrides_kw: dict[str, Any] | None = None,
    hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
579
580
581
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
582

583
    is_gguf = check_gguf_file(model)
584
585
586
587
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        model = Path(model).parent

588
    if config_format == "auto":
589
        try:
590
            if is_gguf or file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
591
                config_format = "hf"
592
            elif file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
593
                config_format = "mistral"
594
595
596
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
597
598
599
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
600
601
                    "in engine args for customized config parser."
                )
602
603
604
605
606
607
608
609
610
611
612

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
613
614
615
                "'params.json'.\n"
                "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
                "   Loading GGUF from a remote repo directly is not yet "
616
617
                "supported.\n"
            ).format(model=model)
618
619

            raise ValueError(error_message) from e
620

621
622
623
624
625
626
627
628
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
        **kwargs,
    )
629
630
631
    # Special architecture mapping check for GGUF models
    if is_gguf:
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
632
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
633
634
635
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

636
637
638
    # Architecture mapping for models without explicit architectures field
    if not config.architectures:
        if config.model_type not in MODEL_MAPPING_NAMES:
639
640
641
642
643
644
645
646
            logger.warning(
                "Model config does not have a top-level 'architectures' field: "
                "expecting `hf_overrides={'architectures': ['...']}` to be passed "
                "in engine args."
            )
        else:
            model_type = MODEL_MAPPING_NAMES[config.model_type]
            config.update({"architectures": [model_type]})
647

648
649
650
651
652
653
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
654
655
656
657
658
659
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
660
661
662

    if quantization_config is not None:
        config.quantization_config = quantization_config
663
        # auto-enable DeepGEMM UE8M0 if model config requests it
664
        scale_fmt = quantization_config.get("scale_fmt", None)
665
        if scale_fmt in ("ue8m0",):
666
667
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
668
                logger.info_once(
669
670
                    (
                        "Detected quantization_config.scale_fmt=%s; "
671
                        "enabling UE8M0 for DeepGEMM."
672
                    ),
673
674
                    scale_fmt,
                )
675
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
676
                logger.warning_once(
677
678
679
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
680
681
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
682
                    ),
683
684
                    scale_fmt,
                )
685

686
687
688
689
690
691
692
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

693
694
    patch_rope_scaling(config)

695
696
697
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

698
    return config
699
700


701
def try_get_local_file(
702
703
    model: str | Path, file_name: str, revision: str | None = "main"
) -> Path | None:
704
705
706
707
708
    file_path = Path(model) / file_name
    if file_path.is_file():
        return file_path
    else:
        try:
709
710
711
            cached_filepath = try_to_load_from_cache(
                repo_id=model, filename=file_name, revision=revision
            )
712
713
            if isinstance(cached_filepath, str):
                return Path(cached_filepath)
714
        except ValueError:
715
716
717
718
            ...
    return None


719
def get_hf_file_to_dict(
720
    file_name: str, model: str | Path, revision: str | None = "main"
721
):
722
    """
723
    Downloads a file from the Hugging Face Hub and returns
724
725
726
727
728
    its contents as a dictionary.

    Parameters:
    - file_name (str): The name of the file to download.
    - model (str): The name of the model on the Hugging Face Hub.
729
    - revision (str): The specific version of the model.
730
731

    Returns:
732
    - config_dict (dict): A dictionary containing
733
734
735
    the contents of the downloaded file.
    """

736
    file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
737

738
    if file_path is None:
739
740
        try:
            hf_hub_file = hf_hub_download(model, file_name, revision=revision)
741
742
        except huggingface_hub.errors.OfflineModeIsEnabled:
            return None
743
744
745
746
747
748
        except (
            RepositoryNotFoundError,
            RevisionNotFoundError,
            EntryNotFoundError,
            LocalEntryNotFoundError,
        ) as e:
749
750
751
752
            logger.debug("File or repository not found in hf_hub_download", e)
            return None
        except HfHubHTTPError as e:
            logger.warning(
753
                "Cannot connect to Hugging Face Hub. Skipping file download for '%s':",
754
                file_name,
755
756
                exc_info=e,
            )
757
758
759
760
            return None
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
761
762
        with open(file_path) as file:
            return json.load(file)
763

764
765
766
    return None


767
@cache
768
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None:
769
    """
770
771
772
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
773
774

    Args:
775
        model: The name of the Hugging Face model.
776
        revision: The specific version of the model to use.
777
            Defaults to 'main'.
778
779

    Returns:
780
        A dictionary containing the pooling type and whether
781
            normalization is used, or None if no pooling configuration is found.
782
783
784
    """

    modules_file_name = "modules.json"
785
786

    modules_dict = None
787
788
789
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
790
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
791
792
793
794

    if modules_dict is None:
        return None

795
796
    logger.info("Found sentence-transformers modules configuration.")

797
798
799
800
801
802
803
804
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
805
    normalize = bool(
806
807
808
809
810
811
812
813
814
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
815
816
817

    if pooling:
        pooling_file_name = "{}/config.json".format(pooling["path"])
818
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
819
        pooling_type_name = next(
820
821
            (item for item, val in pooling_dict.items() if val is True), None
        )
822
823
824
825

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

826
        logger.info("Found pooling configuration.")
827
828
829
830
831
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


832
def get_pooling_config_name(pooling_name: str) -> str | None:
833
834
835
836
837
838
839
840
841
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

842
    supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"]
843
844
    pooling_type_name = pooling_name.upper()

845
846
847
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

848
    raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
849
850


851
@cache
852
def get_sentence_transformer_tokenizer_config(
853
    model: str | Path, revision: str | None = "main"
854
):
855
    """
856
    Returns the tokenization configuration dictionary for a
857
858
859
    given Sentence Transformer BERT model.

    Parameters:
860
    - model (str|Path): The name of the Sentence Transformer
861
862
863
864
865
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
866
    - dict: A dictionary containing the configuration parameters
867
868
    for the Sentence Transformer BERT model.
    """
869
870
871
872
873
874
875
876
877
878
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
879
880

    for config_file in sentence_transformer_config_files:
881
882
883
884
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
885
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
886
887
            if encoder_dict:
                break
888

889
    if not encoder_dict and not Path(model).is_absolute():
890
891
        try:
            # If model is on HuggingfaceHub, get the repo files
892
893
894
            repo_files = list_repo_files(
                model, revision=revision, token=_get_hf_token()
            )
895
        except Exception:
896
897
898
899
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
900
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
901
902
903
                if encoder_dict:
                    break

904
905
906
    if not encoder_dict:
        return None

907
908
    logger.info("Found sentence-transformers tokenize configuration.")

909
910
911
912
913
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


914
def maybe_register_config_serialize_by_value() -> None:
915
916
    """Try to register HF model configuration class to serialize by value

917
918
919
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
920

921
    Examples:
922

923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
947
948
    try:
        import transformers_modules
949

950
        transformers_modules_available = True
951
    except ImportError:
952
        transformers_modules_available = False
953
954
955
956
957

    try:
        import multiprocessing
        import pickle

958
959
        import cloudpickle

960
        from vllm.config import VllmConfig
961

962
963
964
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
965
        def _reduce_config(config: VllmConfig):
966
            return (pickle.loads, (cloudpickle.dumps(config),))
967

968
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
969

970
971
972
973
974
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
975
            from vllm.v1.executor.ray_utils import ray
976

977
978
979
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

980
981
982
983
984
985
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
986
987
            exc_info=e,
        )
988
989


990
def get_hf_image_processor_config(
991
992
993
    model: str | Path,
    hf_token: bool | str | None = None,
    revision: str | None = None,
994
    **kwargs,
995
) -> dict[str, Any]:
996
    # ModelScope does not provide an interface for image_processor
997
    if envs.VLLM_USE_MODELSCOPE:
998
        return dict()
999
    # Separate model folder from file path for GGUF models
1000
    if check_gguf_file(model):
1001
        model = Path(model).parent
1002
1003
1004
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
1005
1006


1007
1008
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
1009
    No op for pure text models.
1010
    """
1011
1012
1013
1014
1015
1016
1017
1018
1019
    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
1020
1021
1022
1023
1024


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
1025
1026
1027
    revision: str | None = None,
    config_format: str | ConfigFormat = "auto",
) -> GenerationConfig | None:
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1039
                config_format=config_format,
1040
1041
1042
1043
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1044
1045


1046
1047
1048
def try_get_safetensors_metadata(
    model: str,
    *,
1049
    revision: str | None = None,
1050
1051
1052
1053
1054
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
1055
        token=_get_hf_token(),
1056
1057
1058
    )

    try:
1059
1060
1061
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1062
1063
    except Exception:
        return None
1064
1065
1066


def try_get_tokenizer_config(
1067
    pretrained_model_name_or_path: str | os.PathLike,
1068
    trust_remote_code: bool,
1069
1070
    revision: str | None = None,
) -> dict[str, Any] | None:
1071
1072
1073
1074
1075
1076
1077
1078
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1079
1080


1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
@cache
def try_get_dense_modules(
    model: str | Path,
    revision: str | None = None,
) -> list[dict[str, Any]] | None:
    try:
        modules = get_hf_file_to_dict("modules.json", model, revision)
        if not modules:
            return None

        if isinstance(modules, dict):
            modules = modules.get("modules", [])

        dense_modules = [
            m for m in modules if m.get("type") == "sentence_transformers.models.Dense"
        ]
        if not dense_modules:
            return None

        layer_configs = []
        for module in dense_modules:
            folder = module.get("path", "")

            config_path = f"{folder}/config.json" if folder else "config.json"
            layer_config = get_hf_file_to_dict(config_path, model, revision)
            if not layer_config:
                continue
            layer_config["folder"] = folder
            layer_configs.append(layer_config)
        return layer_configs
    except Exception:
        return None


1115
1116
1117
def get_safetensors_params_metadata(
    model: str,
    *,
1118
    revision: str | None = None,
1119
1120
1121
1122
1123
1124
1125
1126
1127
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1128
1129
1130
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1143
1144
1145
1146
1147
1148
1149
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1150
1151
            f"and if the config file exists."
        )
1152
1153
1154
1155
1156
1157
1158
1159
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1160
1161
1162
1163
1164
1165
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1166
1167
1168
1169
1170
1171
1172
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1173
1174
            exc_info=e,
        )
1175
1176

    return max_position_embeddings
1177
1178


1179
def get_model_path(model: str | Path, revision: str | None = None):
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
    if os.path.exists(model):
        return model
    assert huggingface_hub.constants.HF_HUB_OFFLINE
    common_kwargs = {
        "local_files_only": huggingface_hub.constants.HF_HUB_OFFLINE,
        "revision": revision,
    }

    if envs.VLLM_USE_MODELSCOPE:
        from modelscope.hub.snapshot_download import snapshot_download
1190

1191
1192
1193
        return snapshot_download(model_id=model, **common_kwargs)

    from huggingface_hub import snapshot_download
1194

1195
    return snapshot_download(repo_id=model, **common_kwargs)
1196
1197


1198
def get_hf_file_bytes(
1199
1200
    file_name: str, model: str | Path, revision: str | None = "main"
) -> bytes | None:
1201
    """Get file contents from HuggingFace repository as bytes."""
1202
    file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
1203
1204

    if file_path is None:
1205
1206
1207
        hf_hub_file = hf_hub_download(
            model, file_name, revision=revision, token=_get_hf_token()
        )
1208
1209
1210
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
1211
        with open(file_path, "rb") as file:
1212
1213
1214
            return file.read()

    return None