config.py 33.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
import enum
import json
6
import os
7
import time
8
from functools import cache, partial
9
from pathlib import Path
10
from typing import Any, Callable, Optional, TypeVar, Union
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
from huggingface_hub import get_safetensors_metadata, hf_hub_download
14
15
from huggingface_hub import list_repo_files as hf_list_repo_files
from huggingface_hub import try_to_load_from_cache
16
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
17
                                   HFValidationError, LocalEntryNotFoundError,
18
19
                                   RepositoryNotFoundError,
                                   RevisionNotFoundError)
20
from transformers import GenerationConfig, PretrainedConfig
21
22
from transformers.models.auto.image_processing_auto import (
    get_image_processor_config)
23
24
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
25
from transformers.models.auto.tokenization_auto import get_tokenizer_config
26
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
27

28
from vllm import envs
29
from vllm.logger import init_logger
30
31
# yapf conflicts with isort for this block
# yapf: disable
32
33
from vllm.transformers_utils.configs import (ChatGLMConfig, DeepseekVLV2Config,
                                             EAGLEConfig, JAISConfig,
34
                                             KimiVLConfig, MedusaConfig,
35
                                             MllamaConfig, MLPSpeculatorConfig,
36
                                             Nemotron_Nano_VL_Config,
37
                                             NemotronConfig, NVLM_D_Config,
38
39
                                             OvisConfig, RWConfig,
                                             SpeculatorsConfig,
40
41
                                             Step3TextConfig, Step3VLConfig,
                                             UltravoxConfig)
42
# yapf: enable
43
from vllm.transformers_utils.configs.mistral import adapt_config_dict
44
from vllm.transformers_utils.utils import check_gguf_file
45

46
if envs.VLLM_USE_MODELSCOPE:
47
48
49
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
50

51
52
MISTRAL_CONFIG_NAME = "params.json"

53
54
logger = init_logger(__name__)

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

def _get_hf_token() -> Optional[str]:
    """
    Get the HuggingFace token from environment variable.

    Returns None if the token is not set, is an empty string, 
    or contains only whitespace.
    This follows the same pattern as huggingface_hub library which
    treats empty string tokens as None to avoid authentication errors.
    """
    token = os.getenv('HF_TOKEN')
    if token and token.strip():
        return token
    return None


71
_CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = {
72
73
74
    "mllama": MllamaConfig
}

75
_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
GoHomeToMacDonal's avatar
GoHomeToMacDonal committed
76
    "chatglm": ChatGLMConfig,
77
    "deepseek_vl_v2": DeepseekVLV2Config,
78
    "kimi_vl": KimiVLConfig,
79
    "Llama_Nemotron_Nano_VL": Nemotron_Nano_VL_Config,
Zhuohan Li's avatar
Zhuohan Li committed
80
81
    "RefinedWeb": RWConfig,  # For tiiuae/falcon-40b(-instruct)
    "RefinedWebModel": RWConfig,  # For tiiuae/falcon-7b(-instruct)
82
    "jais": JAISConfig,
83
    "mlp_speculator": MLPSpeculatorConfig,
84
    "medusa": MedusaConfig,
85
    "eagle": EAGLEConfig,
86
    "speculators": SpeculatorsConfig,
87
    "nemotron": NemotronConfig,
88
    "NVLM_D": NVLM_D_Config,
89
    "ovis": OvisConfig,
90
    "ultravox": UltravoxConfig,
Song's avatar
Song committed
91
92
    "step3_vl": Step3VLConfig,
    "step3_text": Step3TextConfig,
93
    **_CONFIG_REGISTRY_OVERRIDE_HF
94
95
}

96
97
98
99
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

100

101
102
103
104
105
106
class ConfigFormat(str, enum.Enum):
    AUTO = "auto"
    HF = "hf"
    MISTRAL = "mistral"


107
108
109
110
111
112
113
114
115
_R = TypeVar("_R")


def with_retry(
    func: Callable[[], _R],
    log_msg: str,
    max_retries: int = 2,
    retry_delay: int = 2,
) -> _R:
116
117
118
119
120
121
122
123
124
125
126
127
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                logger.error("%s: %s", log_msg, e)
                raise
            logger.error("%s: %s, retrying %d of %d", log_msg, e, attempt + 1,
                         max_retries)
            time.sleep(retry_delay)
            retry_delay *= 2

128
129
    raise AssertionError("Should not be reached")

130
131
132
133
134
135
136
137
138
139
140

# @cache doesn't cache exceptions
@cache
def list_repo_files(
    repo_id: str,
    *,
    revision: Optional[str] = None,
    repo_type: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> list[str]:

141
142
143
144
145
146
147
148
    def lookup_files() -> list[str]:
        # directly list files if model is local
        if (local_path := Path(repo_id)).exists():
            return [
                str(file.relative_to(local_path))
                for file in local_path.rglob('*') if file.is_file()
            ]
        # if model is remote, use hf_hub api to list files
149
        try:
150
            if envs.VLLM_USE_MODELSCOPE:
151
152
153
154
                from vllm.transformers_utils.utils import (
                    modelscope_list_repo_files)
                return modelscope_list_repo_files(repo_id,
                                                  revision=revision,
155
156
157
                                                  token=os.getenv(
                                                      "MODELSCOPE_API_TOKEN",
                                                      None))
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
            return hf_list_repo_files(repo_id,
                                      revision=revision,
                                      repo_type=repo_type,
                                      token=token)
        except huggingface_hub.errors.OfflineModeIsEnabled:
            # Don't raise in offline mode,
            # all we know is that we don't have this
            # file cached.
            return []

    return with_retry(lookup_files, "Error retrieving file list")


def file_exists(
    repo_id: str,
    file_name: str,
    *,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> bool:
    file_list = list_repo_files(repo_id,
                                repo_type=repo_type,
                                revision=revision,
                                token=token)
    return file_name in file_list


# In offline mode the result can be a false negative
187
188
def file_or_path_exists(model: Union[str, Path], config_name: str,
                        revision: Optional[str]) -> bool:
189
190
    if (local_path := Path(model)).exists():
        return (local_path / config_name).is_file()
191

Joe Runde's avatar
Joe Runde committed
192
193
194
195
196
197
198
199
200
201
    # Offline mode support: Check if config file is cached already
    cached_filepath = try_to_load_from_cache(repo_id=model,
                                             filename=config_name,
                                             revision=revision)
    if isinstance(cached_filepath, str):
        # The config file exists in cache- we can continue trying to load
        return True

    # NB: file_exists will only check for the existence of the config file on
    # hf_hub. This will fail in offline mode.
202
203

    # Call HF to check if the file exists
204
205
206
    return file_exists(str(model),
                       config_name,
                       revision=revision,
207
                       token=_get_hf_token())
208
209


210
211
212
213
214
215
216
217
218
219
220
def patch_rope_scaling(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        patch_rope_scaling(text_config)

    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is not None:
        patch_rope_scaling_dict(rope_scaling)


221
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
222
223
224
225
226
227
228
229
230
    if "rope_type" in rope_scaling and "type" in rope_scaling:
        rope_type = rope_scaling["rope_type"]
        rope_type_legacy = rope_scaling["type"]
        if rope_type != rope_type_legacy:
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
                "You should only specify one of them.")

231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
    if "rope_type" not in rope_scaling and "type" in rope_scaling:
        rope_scaling["rope_type"] = rope_scaling["type"]
        logger.info("Replacing legacy 'type' key with 'rope_type'")

    if "rope_type" not in rope_scaling:
        raise ValueError("rope_scaling should have a 'rope_type' key")

    if rope_scaling["rope_type"] == "su":
        rope_scaling["rope_type"] = "longrope"
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
    elif rope_scaling["rope_type"] == "mrope":
        assert "mrope_section" in rope_scaling
        rope_scaling["rope_type"] = "default"
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


247
def _uses_mrope(config: PretrainedConfig) -> bool:
248
249
250
251
252
253
254
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return False

    return "mrope_section" in rope_scaling


255
256
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
257
258
    return _uses_mrope(config) or _uses_mrope(
        config.get_text_config()) or thinker_uses_mrope(config)
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


274
275
276
277
278
279
280
281
282
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        return is_encoder_decoder(text_config)

    return getattr(config, "is_encoder_decoder", False)


283
284
285
286
287
288
289
290
291
292
293
def is_interleaved(config: PretrainedConfig) -> bool:
    """
    Detect if the model with this config is used with interleaved attention.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
        interleaved_types = {"full_attention", "sliding_attention"}
        return interleaved_types.issubset(layer_types)
    return False


294
295
296
297
298
299
300
301
302
303
304
305
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
            delattr(config, old_attr)
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr,
                         new_attr)
    return config


306
def maybe_override_with_speculators_target_model(
307
308
309
310
311
312
    model: str,
    tokenizer: str,
    trust_remote_code: bool,
    revision: Optional[str] = None,
    **kwargs,
) -> tuple[str, str]:
313
314
315
    """
    If running a speculators config, override running model with target model
    """
316
317
318
319
320
321
    is_gguf = check_gguf_file(model)
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        gguf_model_repo = Path(model).parent
    else:
        gguf_model_repo = None
322
    config_dict, _ = PretrainedConfig.get_config_dict(
323
        model if gguf_model_repo is None else gguf_model_repo,
324
325
326
        revision=revision,
        trust_remote_code=trust_remote_code,
        token=_get_hf_token(),
327
        **kwargs,
328
    )
329
    spec_config = config_dict.get("speculators_config", None)
330
331
332
333
334
335
    # Return the target model
    if spec_config is not None:
        model = tokenizer = spec_config["verifier"]["name_or_path"]
    return model, tokenizer


336
337
338
339
340
def get_config(
    model: Union[str, Path],
    trust_remote_code: bool,
    revision: Optional[str] = None,
    code_revision: Optional[str] = None,
341
    config_format: ConfigFormat = ConfigFormat.AUTO,
342
343
344
    hf_overrides_kw: Optional[dict[str, Any]] = None,
    hf_overrides_fn: Optional[Callable[[PretrainedConfig],
                                       PretrainedConfig]] = None,
345
346
347
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
348

349
    is_gguf = check_gguf_file(model)
350
351
352
353
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        model = Path(model).parent

354
    if config_format == ConfigFormat.AUTO:
355
356
357
358
359
360
361
362
        try:
            if is_gguf or file_or_path_exists(
                    model, HF_CONFIG_NAME, revision=revision):
                config_format = ConfigFormat.HF
            elif file_or_path_exists(model,
                                     MISTRAL_CONFIG_NAME,
                                     revision=revision):
                config_format = ConfigFormat.MISTRAL
363
364
365
366
367
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
                    "Ensure your model has either config.json (HF format) "
                    "or params.json (Mistral format).")
368
369
370
371
372
373
374
375
376
377
378

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
379
380
381
382
                "'params.json'.\n"
                "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
                "   Loading GGUF from a remote repo directly is not yet "
                "supported.\n").format(model=model)
383
384

            raise ValueError(error_message) from e
385
386
387

    if config_format == ConfigFormat.HF:
        config_dict, _ = PretrainedConfig.get_config_dict(
388
389
390
            model,
            revision=revision,
            code_revision=code_revision,
391
            token=_get_hf_token(),
392
393
            **kwargs,
        )
394
395
        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
396
397
398
399
        if model_type is None:
            model_type = "speculators" if config_dict.get(
                "speculators_config") is not None else model_type

400
401
        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
402
403
404
405
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
406
                token=_get_hf_token(),
407
408
                **kwargs,
            )
409
410
411
412
413
414
415
        else:
            try:
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
416
                    token=_get_hf_token(),
417
418
419
                    # some old custom model's config needs
                    # `has_no_defaults_at_init=True` to work.
                    has_no_defaults_at_init=trust_remote_code,
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
                    **kwargs,
                )
            except ValueError as e:
                if (not trust_remote_code
                        and "requires you to execute the configuration file"
                        in str(e)):
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
                        "`--trust-remote-code` flag in the CLI.")
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
435
        config = _maybe_remap_hf_config_attrs(config)
436

437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
        # Phi4Flash misuses this config as list[int]. Convert it to int and add
        # the layer_types list[str] to make it HF compatible
        if (config.model_type == "phi4flash"):
            # TODO: Remove after the following PR is merged:
            # https://huggingface.co/microsoft/Phi-4-mini-flash-reasoning/discussions/6
            if not hasattr(config, "layer_types"):
                config.layer_types = [
                    "sliding_attention" if i < config.num_hidden_layers // 2
                    and i % 2 == 1 else "full_attention"
                    for i in range(config.num_hidden_layers)
                ]
            # TODO: Remove after the following PR is merged:
            # https://huggingface.co/microsoft/Phi-4-mini-flash-reasoning/discussions/7
            if isinstance(config.sliding_window, list):
                config.sliding_window = next(
                    filter(None, config.sliding_window), None)

454
    elif config_format == ConfigFormat.MISTRAL:
455
456
457
458
459
460
461
462
463
464
        # This function loads a params.json config which
        # should be used when loading models in mistral format
        config_dict = _download_mistral_config_file(model, revision)
        if (max_position_embeddings :=
                config_dict.get("max_position_embeddings")) is None:
            max_position_embeddings = _maybe_retrieve_max_pos_from_hf(
                model, revision, **kwargs)
            config_dict["max_position_embeddings"] = max_position_embeddings

        config = adapt_config_dict(config_dict)
465
466
467
468
469
470
471
472
473
474
475
476

        # Mistral configs may define sliding_window as list[int]. Convert it
        # to int and add the layer_types list[str] to make it HF compatible
        if ((sliding_window := getattr(config, "sliding_window", None))
                and isinstance(sliding_window, list)):
            pattern_repeats = config.num_hidden_layers // len(sliding_window)
            layer_types = sliding_window * pattern_repeats
            config.layer_types = [
                "full_attention" if layer_type is None else "sliding_attention"
                for layer_type in layer_types
            ]
            config.sliding_window = next(filter(None, sliding_window), None)
477
    else:
478
479
480
481
482
483
484
485
        supported_formats = [
            fmt.value for fmt in ConfigFormat if fmt != ConfigFormat.AUTO
        ]
        raise ValueError(
            f"Unsupported config format: {config_format}. "
            f"Supported formats are: {', '.join(supported_formats)}. "
            f"Ensure your model uses one of these configuration formats "
            f"or specify the correct format explicitly.")
486
487
488
489
490
491
492
493
494

    # Special architecture mapping check for GGUF models
    if is_gguf:
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
            raise RuntimeError(
                f"Can't get gguf config for {config.model_type}.")
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

495
496
497
498
499
500
501
502
503
504
505
506
507
508
    # ModelOpt 0.31.0 and after saves the quantization config in the model
    # config file.
    quantization_config = config_dict.get("quantization_config", None)

    # ModelOpt 0.29.0 and before saves the quantization config in a separate
    # "hf_quant_config.json" in the same directory as the model config file.
    if quantization_config is None \
        and file_or_path_exists(model, "hf_quant_config.json", revision):
        quantization_config = get_hf_file_to_dict("hf_quant_config.json",
                                                  model, revision)

    if quantization_config is not None:
        config.quantization_config = quantization_config

509
510
511
512
513
514
515
    if hf_overrides_kw:
        logger.debug("Overriding HF config with %s", hf_overrides_kw)
        config.update(hf_overrides_kw)
    if hf_overrides_fn:
        logger.debug("Overriding HF config with %s", hf_overrides_fn)
        config = hf_overrides_fn(config)

516
517
    patch_rope_scaling(config)

518
519
520
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

521
    return config
522
523


524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
def try_get_local_file(model: Union[str, Path],
                       file_name: str,
                       revision: Optional[str] = 'main') -> Optional[Path]:
    file_path = Path(model) / file_name
    if file_path.is_file():
        return file_path
    else:
        try:
            cached_filepath = try_to_load_from_cache(repo_id=model,
                                                     filename=file_name,
                                                     revision=revision)
            if isinstance(cached_filepath, str):
                return Path(cached_filepath)
        except HFValidationError:
            ...
    return None


542
543
def get_hf_file_to_dict(file_name: str,
                        model: Union[str, Path],
544
                        revision: Optional[str] = 'main'):
545
    """
546
    Downloads a file from the Hugging Face Hub and returns
547
548
549
550
551
    its contents as a dictionary.

    Parameters:
    - file_name (str): The name of the file to download.
    - model (str): The name of the model on the Hugging Face Hub.
552
    - revision (str): The specific version of the model.
553
554

    Returns:
555
    - config_dict (dict): A dictionary containing
556
557
558
    the contents of the downloaded file.
    """

559
560
561
562
    file_path = try_get_local_file(model=model,
                                   file_name=file_name,
                                   revision=revision)

563
    if file_path is None:
564
565
        try:
            hf_hub_file = hf_hub_download(model, file_name, revision=revision)
566
567
        except huggingface_hub.errors.OfflineModeIsEnabled:
            return None
568
569
570
571
572
573
574
575
576
577
578
579
580
581
        except (RepositoryNotFoundError, RevisionNotFoundError,
                EntryNotFoundError, LocalEntryNotFoundError) as e:
            logger.debug("File or repository not found in hf_hub_download", e)
            return None
        except HfHubHTTPError as e:
            logger.warning(
                "Cannot connect to Hugging Face Hub. Skipping file "
                "download for '%s':",
                file_name,
                exc_info=e)
            return None
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
582
583
        with open(file_path) as file:
            return json.load(file)
584

585
586
587
    return None


588
@cache
589
def get_pooling_config(model: str, revision: Optional[str] = 'main'):
590
    """
591
592
593
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
594
595
596

    Args:
        model (str): The name of the Hugging Face model.
597
        revision (str, optional): The specific version
598
599
600
        of the model to use. Defaults to 'main'.

    Returns:
601
        dict: A dictionary containing the pooling
602
603
604
605
        type and whether normalization is used.
    """

    modules_file_name = "modules.json"
606
607
608
609
610
611

    modules_dict = None
    if file_or_path_exists(model=model,
                           config_name=modules_file_name,
                           revision=revision):
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
612
613
614
615

    if modules_dict is None:
        return None

616
617
    logger.info("Found sentence-transformers modules configuration.")

618
619
620
621
622
623
624
625
626
627
628
    pooling = next((item for item in modules_dict
                    if item["type"] == "sentence_transformers.models.Pooling"),
                   None)
    normalize = bool(
        next((item for item in modules_dict
              if item["type"] == "sentence_transformers.models.Normalize"),
             False))

    if pooling:

        pooling_file_name = "{}/config.json".format(pooling["path"])
629
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
630
631
632
633
634
635
        pooling_type_name = next(
            (item for item, val in pooling_dict.items() if val is True), None)

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

636
        logger.info("Found pooling configuration.")
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

    supported_pooling_types = ['LAST', 'ALL', 'CLS', 'STEP', 'MEAN']
    pooling_type_name = pooling_name.upper()

655
656
657
658
659
    if pooling_type_name in supported_pooling_types:
        return pooling_type_name

    raise NotImplementedError(
        f"Pooling type {pooling_type_name} not supported")
660
661


662
@cache
663
def get_sentence_transformer_tokenizer_config(model: Union[str, Path],
664
665
                                              revision: Optional[str] = 'main'
                                              ):
666
    """
667
    Returns the tokenization configuration dictionary for a
668
669
670
    given Sentence Transformer BERT model.

    Parameters:
671
    - model (str|Path): The name of the Sentence Transformer
672
673
674
675
676
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
677
    - dict: A dictionary containing the configuration parameters
678
679
    for the Sentence Transformer BERT model.
    """
680
681
682
683
684
685
686
687
688
689
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
690
691
692
693
694
695

    for config_file in sentence_transformer_config_files:
        if try_get_local_file(model=model,
                              file_name=config_file,
                              revision=revision) is not None:
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
696
697
            if encoder_dict:
                break
698

699
    if not encoder_dict and not Path(model).is_absolute():
700
701
702
703
        try:
            # If model is on HuggingfaceHub, get the repo files
            repo_files = list_repo_files(model,
                                         revision=revision,
704
                                         token=_get_hf_token())
705
        except Exception:
706
707
708
709
710
711
712
713
714
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
                encoder_dict = get_hf_file_to_dict(config_name, model,
                                                   revision)
                if encoder_dict:
                    break

715
716
717
    if not encoder_dict:
        return None

718
719
    logger.info("Found sentence-transformers tokenize configuration.")

720
721
722
723
724
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


725
def maybe_register_config_serialize_by_value() -> None:
726
727
    """Try to register HF model configuration class to serialize by value

728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
        If trust_remote_code is set, and the model's config file specifies an
        `AutoConfig` class, then the config class is typically an instance of
        a custom class imported from the HF modules cache.

        Examples:

        >>> from transformers import AutoConfig
        >>> klass = AutoConfig.from_pretrained('meta-llama/Meta-Llama-3-8B', trust_remote_code=True)
        >>> klass.__class__ # transformers.models.llama.configuration_llama.LlamaConfig
        >>> import transformers_modules # error, not initialized
        >>> klass = AutoConfig.from_pretrained('deepseek-ai/DeepSeek-V2.5', trust_remote_code=True)
        >>> import transformers_modules # success, initialized
        >>> klass.__class__ # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

        In the DeepSeek example, the config class is an instance of a custom
        class that is not serializable by default. This class will not be
        importable in spawned workers, and won't exist at all on
        other nodes, which breaks serialization of the config.
746
747
748
749

        In this function we tell the cloudpickle serialization library to pass
        instances of these generated classes by value instead of by reference,
        i.e. the class definition is serialized along with its data so that the
750
        class module does not need to be importable on the receiving end.
751
752

        See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
753
    """ # noqa
754
755
    try:
        import transformers_modules
756
        transformers_modules_available = True
757
    except ImportError:
758
        transformers_modules_available = False
759
760
761
762
763

    try:
        import multiprocessing
        import pickle

764
765
        import cloudpickle

766
        from vllm.config import VllmConfig
767

768
769
770
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
771
772
        def _reduce_config(config: VllmConfig):
            return (pickle.loads, (cloudpickle.dumps(config), ))
773

774
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
775

776
777
778
779
780
781
782
783
784
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
            from vllm.executor.ray_utils import ray
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

785
786
787
788
789
790
791
792
793
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
            exc_info=e)


794
795
def get_hf_image_processor_config(
    model: Union[str, Path],
796
    hf_token: Optional[Union[bool, str]] = None,
797
798
    revision: Optional[str] = None,
    **kwargs,
799
) -> dict[str, Any]:
800
    # ModelScope does not provide an interface for image_processor
801
    if envs.VLLM_USE_MODELSCOPE:
802
        return dict()
803
    # Separate model folder from file path for GGUF models
804
    if check_gguf_file(model):
805
        model = Path(model).parent
806
807
808
809
    return get_image_processor_config(model,
                                      token=hf_token,
                                      revision=revision,
                                      **kwargs)
810
811


812
813
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
814
    No op for pure text models.
815
    """
816
817
818
819
820
821
822
823
824
    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
    revision: Optional[str] = None,
) -> Optional[GenerationConfig]:
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
847
848


849
850
851
852
853
854
855
856
857
def try_get_safetensors_metadata(
    model: str,
    *,
    revision: Optional[str] = None,
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
858
        token=_get_hf_token(),
859
860
861
862
863
864
865
    )

    try:
        return with_retry(get_safetensors_metadata_partial,
                          "Error retrieving safetensors")
    except Exception:
        return None
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880


def try_get_tokenizer_config(
    pretrained_model_name_or_path: Union[str, os.PathLike],
    trust_remote_code: bool,
    revision: Optional[str] = None,
) -> Optional[dict[str, Any]]:
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912


def _download_mistral_config_file(model, revision) -> dict:
    config_file_name = "params.json"
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
            f"and if the config file exists.")
    assert isinstance(config_dict, dict)
    return config_dict


def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
    max_position_embeddings = 128_000
    try:
        trust_remote_code_val = kwargs.get("trust_remote_code", False)
        hf_config = get_config(model=model,
                               trust_remote_code=trust_remote_code_val,
                               revision=revision,
                               config_format=ConfigFormat.HF)
        if hf_value := hf_config.get_text_config().max_position_embeddings:
            max_position_embeddings = hf_value
    except Exception as e:
        logger.warning(
            "The params.json file is missing 'max_position_embeddings'"
            " and could not get a value from the HF config."
            " Defaulting to 128000",
            exc_info=e)

    return max_position_embeddings