config.py 38.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import json
5
import os
6
import time
7
from dataclasses import asdict
8
from functools import cache, partial
9
from pathlib import Path
10
from typing import Any, Callable, Literal, Optional, TypeVar, Union
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
14
15
16
17
from huggingface_hub import (
    get_safetensors_metadata,
    hf_hub_download,
    try_to_load_from_cache,
)
18
from huggingface_hub import list_repo_files as hf_list_repo_files
19
20
21
22
23
24
25
from huggingface_hub.utils import (
    EntryNotFoundError,
    HfHubHTTPError,
    LocalEntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
)
26
from transformers import GenerationConfig, PretrainedConfig
27
28
from transformers.models.auto.image_processing_auto import get_image_processor_config
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
29
from transformers.models.auto.tokenization_auto import get_tokenizer_config
30
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
31

32
from vllm import envs
33
from vllm.logger import init_logger
34
from vllm.transformers_utils.config_parser_base import ConfigParserBase
35
36
37
38
from vllm.transformers_utils.utils import (
    check_gguf_file,
    parse_safetensors_file_metadata,
)
39

40
if envs.VLLM_USE_MODELSCOPE:
41
42
43
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
44

45
46
MISTRAL_CONFIG_NAME = "params.json"

47
48
logger = init_logger(__name__)

49
50
51
52
53

def _get_hf_token() -> Optional[str]:
    """
    Get the HuggingFace token from environment variable.

54
    Returns None if the token is not set, is an empty string,
55
56
57
58
    or contains only whitespace.
    This follows the same pattern as huggingface_hub library which
    treats empty string tokens as None to avoid authentication errors.
    """
59
    token = os.getenv("HF_TOKEN")
60
61
62
63
64
    if token and token.strip():
        return token
    return None


65
66
67
class LazyConfigDict(dict):
    def __getitem__(self, key):
        import vllm.transformers_utils.configs as configs
68

69
70
71
72
73
74
        return getattr(configs, super().__getitem__(key))


_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
    chatglm="ChatGLMConfig",
    deepseek_vl_v2="DeepseekVLV2Config",
75
76
    deepseek_v3="DeepseekV3Config",
    deepseek_v32="DeepseekV3Config",
77
    flex_olmo="FlexOlmoConfig",
78
79
80
81
82
83
84
    kimi_vl="KimiVLConfig",
    Llama_Nemotron_Nano_VL="Nemotron_Nano_VL_Config",
    RefinedWeb="RWConfig",  # For tiiuae/falcon-40b(-instruct)
    RefinedWebModel="RWConfig",  # For tiiuae/falcon-7b(-instruct)
    jais="JAISConfig",
    mlp_speculator="MLPSpeculatorConfig",
    medusa="MedusaConfig",
85
    midashenglm="MiDashengLMConfig",
86
87
88
    eagle="EAGLEConfig",
    speculators="SpeculatorsConfig",
    nemotron="NemotronConfig",
89
    olmo3="Olmo3Config",
90
91
92
93
    ovis="OvisConfig",
    ultravox="UltravoxConfig",
    step3_vl="Step3VLConfig",
    step3_text="Step3TextConfig",
94
    qwen3_next="Qwen3NextConfig",
Paul Pak's avatar
Paul Pak committed
95
    lfm2_moe="Lfm2MoeConfig",
96
)
97

98
99
100
101
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

102
_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
103
104
    "internvl_chat": {"has_no_defaults_at_init": True},
    "NVLM_D": {"has_no_defaults_at_init": True},
105
106
}

107

108
class HFConfigParser(ConfigParserBase):
109
110
111
112
113
114
115
116
    def parse(
        self,
        model: Union[str, Path],
        trust_remote_code: bool,
        revision: Optional[str] = None,
        code_revision: Optional[str] = None,
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
117
118
119
120
121
122
123
124
125
126
127
        kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
        config_dict, _ = PretrainedConfig.get_config_dict(
            model,
            revision=revision,
            code_revision=code_revision,
            token=_get_hf_token(),
            **kwargs,
        )
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type is None:
128
129
130
131
132
            model_type = (
                "speculators"
                if config_dict.get("speculators_config") is not None
                else model_type
            )
133
134
135
136
137
138
139
140
141
142
143
144

        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
                token=_get_hf_token(),
                **kwargs,
            )
        else:
            try:
145
                kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
146
147
148
149
150
151
152
153
154
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
                    token=_get_hf_token(),
                    **kwargs,
                )
            except ValueError as e:
155
156
157
158
                if (
                    not trust_remote_code
                    and "requires you to execute the configuration file" in str(e)
                ):
159
160
161
162
163
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
164
165
                        "`--trust-remote-code` flag in the CLI."
                    )
166
167
168
169
170
171
172
173
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
        config = _maybe_remap_hf_config_attrs(config)
        return config_dict, config


class MistralConfigParser(ConfigParserBase):
174
175
176
177
178
179
180
181
    def parse(
        self,
        model: Union[str, Path],
        trust_remote_code: bool,
        revision: Optional[str] = None,
        code_revision: Optional[str] = None,
        **kwargs,
    ) -> tuple[dict, PretrainedConfig]:
182
183
184
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
185
186
187
        if (
            max_position_embeddings := config_dict.get("max_position_embeddings")
        ) is None:
188
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
189
190
                model, revision, **kwargs
            )
191
192
193
194
195
196
197
198
            config_dict["max_position_embeddings"] = max_position_embeddings

        from vllm.transformers_utils.configs.mistral import adapt_config_dict

        config = adapt_config_dict(config_dict)

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
199
200
201
        if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
            sliding_window, list
        ):
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)

        return config_dict, config


_CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = {
    "hf": HFConfigParser,
    "mistral": MistralConfigParser,
}

ConfigFormat = Literal[
    "auto",
    "hf",
    "mistral",
]


def get_config_parser(config_format: str) -> ConfigParserBase:
    """Get the config parser for a given config format."""
    if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER:
        raise ValueError(f"Unknown config format `{config_format}`.")
    return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]()


def register_config_parser(config_format: str):
    """Register a customized vllm config parser.
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
     When a config format is not supported by vllm, you can register a customized
    config parser to support it.
     Args:
         config_format (str): The config parser format name.
     Examples:

         >>> from vllm.transformers_utils.config import (get_config_parser,
                                                         register_config_parser)
         >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase
         >>>
         >>> @register_config_parser("custom_config_parser")
         ... class CustomConfigParser(ConfigParserBase):
         ...     def parse(
         ...         self,
         ...         model: Union[str, Path],
         ...         trust_remote_code: bool,
         ...         revision: Optional[str] = None,
         ...         code_revision: Optional[str] = None,
         ...         **kwargs,
         ...     ) -> tuple[dict, PretrainedConfig]:
         ...         raise NotImplementedError
         >>>
         >>> type(get_config_parser("custom_config_parser"))
         <class 'CustomConfigParser'>
258
259
260
261
262
263
    """  # noqa: E501

    def _wrapper(config_parser_cls):
        if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER:
            logger.warning(
                "Config format `%s` is already registered, and will be "
264
265
266
267
                "overwritten by the new parser class `%s`.",
                config_format,
                config_parser_cls,
            )
268
        if not issubclass(config_parser_cls, ConfigParserBase):
269
270
271
            raise ValueError(
                "The config parser must be a subclass of `ConfigParserBase`."
            )
272
        _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls
273
274
275
276
277
        logger.info(
            "Registered config parser `%s` with config format `%s`",
            config_parser_cls,
            config_format,
        )
278
279
280
        return config_parser_cls

    return _wrapper
281
282


283
284
285
286
287
288
289
290
291
_R = TypeVar("_R")


def with_retry(
    func: Callable[[], _R],
    log_msg: str,
    max_retries: int = 2,
    retry_delay: int = 2,
) -> _R:
292
293
294
295
296
297
298
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                logger.error("%s: %s", log_msg, e)
                raise
299
300
301
            logger.error(
                "%s: %s, retrying %d of %d", log_msg, e, attempt + 1, max_retries
            )
302
303
304
            time.sleep(retry_delay)
            retry_delay *= 2

305
306
    raise AssertionError("Should not be reached")

307
308
309
310
311
312
313
314
315
316

# @cache doesn't cache exceptions
@cache
def list_repo_files(
    repo_id: str,
    *,
    revision: Optional[str] = None,
    repo_type: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> list[str]:
317
318
319
320
321
    def lookup_files() -> list[str]:
        # directly list files if model is local
        if (local_path := Path(repo_id)).exists():
            return [
                str(file.relative_to(local_path))
322
323
                for file in local_path.rglob("*")
                if file.is_file()
324
325
            ]
        # if model is remote, use hf_hub api to list files
326
        try:
327
            if envs.VLLM_USE_MODELSCOPE:
328
329
330
331
332
333
334
335
336
337
                from vllm.transformers_utils.utils import modelscope_list_repo_files

                return modelscope_list_repo_files(
                    repo_id,
                    revision=revision,
                    token=os.getenv("MODELSCOPE_API_TOKEN", None),
                )
            return hf_list_repo_files(
                repo_id, revision=revision, repo_type=repo_type, token=token
            )
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
        except huggingface_hub.errors.OfflineModeIsEnabled:
            # Don't raise in offline mode,
            # all we know is that we don't have this
            # file cached.
            return []

    return with_retry(lookup_files, "Error retrieving file list")


def file_exists(
    repo_id: str,
    file_name: str,
    *,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> bool:
355
356
357
    file_list = list_repo_files(
        repo_id, repo_type=repo_type, revision=revision, token=token
    )
358
359
360
361
    return file_name in file_list


# In offline mode the result can be a false negative
362
363
364
def file_or_path_exists(
    model: Union[str, Path], config_name: str, revision: Optional[str]
) -> bool:
365
366
    if (local_path := Path(model)).exists():
        return (local_path / config_name).is_file()
367

Joe Runde's avatar
Joe Runde committed
368
    # Offline mode support: Check if config file is cached already
369
370
371
    cached_filepath = try_to_load_from_cache(
        repo_id=model, filename=config_name, revision=revision
    )
Joe Runde's avatar
Joe Runde committed
372
373
374
375
376
377
    if isinstance(cached_filepath, str):
        # The config file exists in cache- we can continue trying to load
        return True

    # NB: file_exists will only check for the existence of the config file on
    # hf_hub. This will fail in offline mode.
378
379

    # Call HF to check if the file exists
380
381
382
    return file_exists(
        str(model), config_name, revision=revision, token=_get_hf_token()
    )
383
384


385
386
387
388
389
390
391
392
393
394
395
def patch_rope_scaling(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        patch_rope_scaling(text_config)

    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is not None:
        patch_rope_scaling_dict(rope_scaling)


396
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
397
398
399
400
401
402
403
    if "rope_type" in rope_scaling and "type" in rope_scaling:
        rope_type = rope_scaling["rope_type"]
        rope_type_legacy = rope_scaling["type"]
        if rope_type != rope_type_legacy:
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
404
405
                "You should only specify one of them."
            )
406

407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
    if "rope_type" not in rope_scaling and "type" in rope_scaling:
        rope_scaling["rope_type"] = rope_scaling["type"]
        logger.info("Replacing legacy 'type' key with 'rope_type'")

    if "rope_type" not in rope_scaling:
        raise ValueError("rope_scaling should have a 'rope_type' key")

    if rope_scaling["rope_type"] == "su":
        rope_scaling["rope_type"] = "longrope"
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
    elif rope_scaling["rope_type"] == "mrope":
        assert "mrope_section" in rope_scaling
        rope_scaling["rope_type"] = "default"
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


423
def _uses_mrope(config: PretrainedConfig) -> bool:
424
425
426
427
428
429
430
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return False

    return "mrope_section" in rope_scaling


431
432
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
433
434
435
436
437
    return (
        _uses_mrope(config)
        or _uses_mrope(config.get_text_config())
        or thinker_uses_mrope(config)
    )
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


453
454
455
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""

456
457
458
    def _is_encoder_decoder(config: PretrainedConfig) -> bool:
        return getattr(config, "is_encoder_decoder", False)

459
    return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config())
460
461


462
463
464
465
466
467
468
469
470
471
472
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
        interleaved_types = {"full_attention", "sliding_attention"}
        return interleaved_types.issubset(layer_types)
    return False


473
474
475
476
477
478
479
480
481
def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str):
    """
    Update kwargs for AutoConfig initialization based on model_type
    """
    if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES:
        kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type])
    return kwargs


482
483
484
485
486
487
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
488
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr)
489
490
491
    return config


492
def maybe_override_with_speculators(
493
494
495
496
    model: str,
    tokenizer: str,
    trust_remote_code: bool,
    revision: Optional[str] = None,
497
    vllm_speculative_config: Optional[dict[str, Any]] = None,
498
    **kwargs,
499
) -> tuple[str, str, Optional[dict[str, Any]]]:
500
    """
501
502
503
504
505
506
507
508
509
510
511
512
513
514
    Resolve model configuration when speculators are detected.

    Checks if the provided model is a speculators model and if so, extracts
    the target model configuration and builds the speculative config.

    Args:
        model: Model name or path
        tokenizer: Tokenizer name or path
        trust_remote_code: Whether to trust remote code
        revision: Model revision
        vllm_speculative_config: Existing vLLM speculative config

    Returns:
        Tuple of (resolved_model, resolved_tokenizer, speculative_config)
515
    """
516
517
518
519
520
521
    is_gguf = check_gguf_file(model)
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
    else:
        gguf_model_repo = None
522
    kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
523
    config_dict, _ = PretrainedConfig.get_config_dict(
524
        model if gguf_model_repo is None else gguf_model_repo,
525
526
527
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
528
        **kwargs,
529
    )
530
531
532
533
534
535
536
    speculators_config = config_dict.get("speculators_config")

    if speculators_config is None:
        # No speculators config found, return original values
        return model, tokenizer, vllm_speculative_config

    # Speculators format detected - process overrides
537
    from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig
538

539
    speculative_config = SpeculatorsConfig.extract_vllm_speculative_config(
540
541
        config_dict=config_dict
    )
542
543

    # Set the draft model to the speculators model
544
    speculative_config["model"] = model
545
546
547
548
549

    # Override model and tokenizer with the verifier model from config
    verifier_model = speculators_config["verifier"]["name_or_path"]
    model = tokenizer = verifier_model

550
    return model, tokenizer, speculative_config
551
552


553
554
555
556
557
def get_config(
    model: Union[str, Path],
    trust_remote_code: bool,
    revision: Optional[str] = None,
    code_revision: Optional[str] = None,
558
    config_format: Union[str, ConfigFormat] = "auto",
559
    hf_overrides_kw: Optional[dict[str, Any]] = None,
560
    hf_overrides_fn: Optional[Callable[[PretrainedConfig], PretrainedConfig]] = None,
561
562
563
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
564

565
    is_gguf = check_gguf_file(model)
566
567
568
569
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        model = Path(model).parent

570
    if config_format == "auto":
571
        try:
572
            if is_gguf or file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
573
                config_format = "hf"
574
            elif file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
575
                config_format = "mistral"
576
577
578
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
579
580
581
                    "With config_format 'auto', ensure your model has either "
                    "config.json (HF format) or params.json (Mistral format). "
                    "Otherwise please specify your_custom_config_format "
582
583
                    "in engine args for customized config parser."
                )
584
585
586
587
588
589
590
591
592
593
594

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
595
596
597
                "'params.json'.\n"
                "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
                "   Loading GGUF from a remote repo directly is not yet "
598
599
                "supported.\n"
            ).format(model=model)
600
601

            raise ValueError(error_message) from e
602

603
604
605
606
607
608
609
610
    config_parser = get_config_parser(config_format)
    config_dict, config = config_parser.parse(
        model,
        trust_remote_code=trust_remote_code,
        revision=revision,
        code_revision=code_revision,
        **kwargs,
    )
611
612
613
    # Special architecture mapping check for GGUF models
    if is_gguf:
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
614
            raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
615
616
617
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

618
619
620
621
622
623
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
624
625
626
627
628
629
    if quantization_config is None and file_or_path_exists(
        model, "hf_quant_config.json", revision
    ):
        quantization_config = get_hf_file_to_dict(
            "hf_quant_config.json", model, revision
        )
630
631
632

    if quantization_config is not None:
        config.quantization_config = quantization_config
633
        # auto-enable DeepGEMM UE8M0 if model config requests it
634
        scale_fmt = quantization_config.get("scale_fmt", None)
635
        if scale_fmt in ("ue8m0",):
636
637
            if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"):
                os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1"
638
                logger.info_once(
639
640
                    (
                        "Detected quantization_config.scale_fmt=%s; "
641
                        "enabling UE8M0 for DeepGEMM."
642
                    ),
643
644
                    scale_fmt,
                )
645
            elif not envs.VLLM_USE_DEEP_GEMM_E8M0:
646
                logger.warning_once(
647
648
649
                    (
                        "Model config requests UE8M0 "
                        "(quantization_config.scale_fmt=%s), but "
650
651
                        "VLLM_USE_DEEP_GEMM_E8M0=0 is set; "
                        "UE8M0 for DeepGEMM disabled."
652
                    ),
653
654
                    scale_fmt,
                )
655

656
657
658
659
660
661
662
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

663
664
    patch_rope_scaling(config)

665
666
667
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

668
    return config
669
670


671
672
673
def try_get_local_file(
    model: Union[str, Path], file_name: str, revision: Optional[str] = "main"
) -> Optional[Path]:
674
675
676
677
678
    file_path = Path(model) / file_name
    if file_path.is_file():
        return file_path
    else:
        try:
679
680
681
            cached_filepath = try_to_load_from_cache(
                repo_id=model, filename=file_name, revision=revision
            )
682
683
            if isinstance(cached_filepath, str):
                return Path(cached_filepath)
684
        except ValueError:
685
686
687
688
            ...
    return None


689
690
691
def get_hf_file_to_dict(
    file_name: str, model: Union[str, Path], revision: Optional[str] = "main"
):
692
    """
693
    Downloads a file from the Hugging Face Hub and returns
694
695
696
697
698
    its contents as a dictionary.

    Parameters:
    - file_name (str): The name of the file to download.
    - model (str): The name of the model on the Hugging Face Hub.
699
    - revision (str): The specific version of the model.
700
701

    Returns:
702
    - config_dict (dict): A dictionary containing
703
704
705
    the contents of the downloaded file.
    """

706
    file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
707

708
    if file_path is None:
709
710
        try:
            hf_hub_file = hf_hub_download(model, file_name, revision=revision)
711
712
        except huggingface_hub.errors.OfflineModeIsEnabled:
            return None
713
714
715
716
717
718
        except (
            RepositoryNotFoundError,
            RevisionNotFoundError,
            EntryNotFoundError,
            LocalEntryNotFoundError,
        ) as e:
719
720
721
722
            logger.debug("File or repository not found in hf_hub_download", e)
            return None
        except HfHubHTTPError as e:
            logger.warning(
723
                "Cannot connect to Hugging Face Hub. Skipping file download for '%s':",
724
                file_name,
725
726
                exc_info=e,
            )
727
728
729
730
            return None
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
731
732
        with open(file_path) as file:
            return json.load(file)
733

734
735
736
    return None


737
@cache
738
def get_pooling_config(model: str, revision: Optional[str] = "main") -> Optional[dict]:
739
    """
740
741
742
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
743
744

    Args:
745
        model: The name of the Hugging Face model.
746
        revision: The specific version of the model to use.
747
            Defaults to 'main'.
748
749

    Returns:
750
        A dictionary containing the pooling type and whether
751
            normalization is used, or None if no pooling configuration is found.
752
753
754
    """

    modules_file_name = "modules.json"
755
756

    modules_dict = None
757
758
759
    if file_or_path_exists(
        model=model, config_name=modules_file_name, revision=revision
    ):
760
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
761
762
763
764

    if modules_dict is None:
        return None

765
766
    logger.info("Found sentence-transformers modules configuration.")

767
768
769
770
771
772
773
774
    pooling = next(
        (
            item
            for item in modules_dict
            if item["type"] == "sentence_transformers.models.Pooling"
        ),
        None,
    )
775
    normalize = bool(
776
777
778
779
780
781
782
783
784
        next(
            (
                item
                for item in modules_dict
                if item["type"] == "sentence_transformers.models.Normalize"
            ),
            False,
        )
    )
785
786
787

    if pooling:
        pooling_file_name = "{}/config.json".format(pooling["path"])
788
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
789
        pooling_type_name = next(
790
791
            (item for item, val in pooling_dict.items() if val is True), None
        )
792
793
794
795

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

796
        logger.info("Found pooling configuration.")
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

812
    supported_pooling_types = ["LAST", "ALL", "CLS", "STEP", "MEAN"]
813
814
    pooling_type_name = pooling_name.upper()

815
816
817
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

818
    raise NotImplementedError(f"Pooling type {pooling_type_name} not supported")
819
820


821
@cache
822
823
824
def get_sentence_transformer_tokenizer_config(
    model: Union[str, Path], revision: Optional[str] = "main"
):
825
    """
826
    Returns the tokenization configuration dictionary for a
827
828
829
    given Sentence Transformer BERT model.

    Parameters:
830
    - model (str|Path): The name of the Sentence Transformer
831
832
833
834
835
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
836
    - dict: A dictionary containing the configuration parameters
837
838
    for the Sentence Transformer BERT model.
    """
839
840
841
842
843
844
845
846
847
848
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
849
850

    for config_file in sentence_transformer_config_files:
851
852
853
854
        if (
            try_get_local_file(model=model, file_name=config_file, revision=revision)
            is not None
        ):
855
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
856
857
            if encoder_dict:
                break
858

859
    if not encoder_dict and not Path(model).is_absolute():
860
861
        try:
            # If model is on HuggingfaceHub, get the repo files
862
863
864
            repo_files = list_repo_files(
                model, revision=revision, token=_get_hf_token()
            )
865
        except Exception:
866
867
868
869
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
870
                encoder_dict = get_hf_file_to_dict(config_name, model, revision)
871
872
873
                if encoder_dict:
                    break

874
875
876
    if not encoder_dict:
        return None

877
878
    logger.info("Found sentence-transformers tokenize configuration.")

879
880
881
882
883
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


884
def maybe_register_config_serialize_by_value() -> None:
885
886
    """Try to register HF model configuration class to serialize by value

887
888
889
    If trust_remote_code is set, and the model's config file specifies an
    `AutoConfig` class, then the config class is typically an instance of
    a custom class imported from the HF modules cache.
890

891
    Examples:
892

893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
    >>> from transformers import AutoConfig
    >>> klass = AutoConfig.from_pretrained(
    ...     "meta-llama/Meta-Llama-3-8B", trust_remote_code=True
    ... )
    >>> klass.__class__  # transformers.models.llama.configuration_llama.LlamaConfig
    >>> import transformers_modules  # error, not initialized
    >>> klass = AutoConfig.from_pretrained(
    ...     "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True
    ... )
    >>> import transformers_modules  # success, initialized
    >>> klass.__class__  # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

    In the DeepSeek example, the config class is an instance of a custom
    class that is not serializable by default. This class will not be
    importable in spawned workers, and won't exist at all on
    other nodes, which breaks serialization of the config.

    In this function we tell the cloudpickle serialization library to pass
    instances of these generated classes by value instead of by reference,
    i.e. the class definition is serialized along with its data so that the
    class module does not need to be importable on the receiving end.

    See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
    """  # noqa
917
918
    try:
        import transformers_modules
919

920
        transformers_modules_available = True
921
    except ImportError:
922
        transformers_modules_available = False
923
924
925
926
927

    try:
        import multiprocessing
        import pickle

928
929
        import cloudpickle

930
        from vllm.config import VllmConfig
931

932
933
934
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
935
        def _reduce_config(config: VllmConfig):
936
            return (pickle.loads, (cloudpickle.dumps(config),))
937

938
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
939

940
941
942
943
944
945
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
            from vllm.executor.ray_utils import ray
946

947
948
949
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

950
951
952
953
954
955
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
956
957
            exc_info=e,
        )
958
959


960
961
def get_hf_image_processor_config(
    model: Union[str, Path],
962
    hf_token: Optional[Union[bool, str]] = None,
963
964
    revision: Optional[str] = None,
    **kwargs,
965
) -> dict[str, Any]:
966
    # ModelScope does not provide an interface for image_processor
967
    if envs.VLLM_USE_MODELSCOPE:
968
        return dict()
969
    # Separate model folder from file path for GGUF models
970
    if check_gguf_file(model):
971
        model = Path(model).parent
972
973
974
    return get_image_processor_config(
        model, token=hf_token, revision=revision, **kwargs
    )
975
976


977
978
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
979
    No op for pure text models.
980
    """
981
982
983
984
985
986
987
988
989
    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
990
991
992
993
994
995


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
    revision: Optional[str] = None,
996
    config_format: Union[str, ConfigFormat] = "auto",
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
) -> Optional[GenerationConfig]:
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
1009
                config_format=config_format,
1010
1011
1012
1013
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
1014
1015


1016
1017
1018
1019
1020
1021
1022
1023
1024
def try_get_safetensors_metadata(
    model: str,
    *,
    revision: Optional[str] = None,
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
1025
        token=_get_hf_token(),
1026
1027
1028
    )

    try:
1029
1030
1031
        return with_retry(
            get_safetensors_metadata_partial, "Error retrieving safetensors"
        )
1032
1033
    except Exception:
        return None
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048


def try_get_tokenizer_config(
    pretrained_model_name_or_path: Union[str, os.PathLike],
    trust_remote_code: bool,
    revision: Optional[str] = None,
) -> Optional[dict[str, Any]]:
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
1049
1050


1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
def get_safetensors_params_metadata(
    model: str,
    *,
    revision: Optional[str] = None,
) -> dict[str, Any]:
    """
    Get the safetensors metadata for remote model repository.
    """
    full_metadata = {}
    if (model_path := Path(model)).exists():
        safetensors_to_check = model_path.glob("*.safetensors")
        full_metadata = {
            param_name: info
1064
1065
1066
            for file_path in safetensors_to_check
            if file_path.is_file()
            for param_name, info in parse_safetensors_file_metadata(file_path).items()
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
        }
    else:
        repo_mt = try_get_safetensors_metadata(model, revision=revision)
        if repo_mt and (files_mt := repo_mt.files_metadata):
            full_metadata = {
                param_name: asdict(info)
                for file_mt in files_mt.values()
                for param_name, info in file_mt.tensors.items()
            }
    return full_metadata


1079
1080
1081
1082
1083
1084
1085
def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
1086
1087
            f"and if the config file exists."
        )
1088
1089
1090
1091
1092
1093
1094
1095
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
1096
1097
1098
1099
1100
1101
        hf_config = get_config(
            model=model,
            trust_remote_code=trust_remote_code_val,
            revision=revision,
            config_format="hf",
        )
1102
1103
1104
1105
1106
1107
1108
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
1109
1110
            exc_info=e,
        )
1111
1112

    return max_position_embeddings
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125


def get_model_path(model: Union[str, Path], revision: Optional[str] = None):
    if os.path.exists(model):
        return model
    assert huggingface_hub.constants.HF_HUB_OFFLINE
    common_kwargs = {
        "local_files_only": huggingface_hub.constants.HF_HUB_OFFLINE,
        "revision": revision,
    }

    if envs.VLLM_USE_MODELSCOPE:
        from modelscope.hub.snapshot_download import snapshot_download
1126

1127
1128
1129
        return snapshot_download(model_id=model, **common_kwargs)

    from huggingface_hub import snapshot_download
1130

1131
    return snapshot_download(repo_id=model, **common_kwargs)
1132
1133


1134
1135
1136
def get_hf_file_bytes(
    file_name: str, model: Union[str, Path], revision: Optional[str] = "main"
) -> Optional[bytes]:
1137
    """Get file contents from HuggingFace repository as bytes."""
1138
    file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
1139
1140

    if file_path is None:
1141
1142
1143
        hf_hub_file = hf_hub_download(
            model, file_name, revision=revision, token=_get_hf_token()
        )
1144
1145
1146
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
1147
        with open(file_path, "rb") as file:
1148
1149
1150
            return file.read()

    return None