config.py 34.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
import enum
import json
6
import os
7
import time
8
from functools import cache, partial
9
from pathlib import Path
10
from typing import Any, Callable, Literal, Optional, TypeVar, Union
Jasmond L's avatar
Jasmond L committed
11

Joe Runde's avatar
Joe Runde committed
12
import huggingface_hub
13
from huggingface_hub import get_safetensors_metadata, hf_hub_download
14
15
from huggingface_hub import list_repo_files as hf_list_repo_files
from huggingface_hub import try_to_load_from_cache
16
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
17
                                   HFValidationError, LocalEntryNotFoundError,
18
19
                                   RepositoryNotFoundError,
                                   RevisionNotFoundError)
20
from torch import nn
21
from transformers import GenerationConfig, PretrainedConfig
22
23
from transformers.models.auto.image_processing_auto import (
    get_image_processor_config)
24
25
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
26
from transformers.models.auto.tokenization_auto import get_tokenizer_config
27
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
28

29
from vllm import envs
30
from vllm.logger import init_logger
31
32
# yapf conflicts with isort for this block
# yapf: disable
33
34
35
from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
                                             DbrxConfig, DeepseekVLV2Config,
                                             EAGLEConfig, ExaoneConfig,
36
37
                                             JAISConfig, KimiVLConfig,
                                             MedusaConfig, MiniMaxText01Config,
38
39
40
                                             MiniMaxVL01Config, MllamaConfig,
                                             MLPSpeculatorConfig, MPTConfig,
                                             NemotronConfig, NVLM_D_Config,
41
                                             OvisConfig, RWConfig,
zhuwenwen's avatar
zhuwenwen committed
42
                                             Step3TextConfig, Step3VLConfig, 
43
                                             SkyworkR1VChatConfig, SolarConfig,
zhuwenwen's avatar
zhuwenwen committed
44
45
46
47
48
49
50
51
52
                                             MMGPTStep1Config,
                                             MMGPTStep1ConfigV2, MPTConfig,
                                             NemotronConfig, NVLM_D_Config,
                                             RWConfig, SkyworkR1VChatConfig,
                                             SolarConfig, Step1AudioConfig,
                                             Step1Config, Step1oConfig,
                                             Step2Config, Step2MiniConfig,
                                             Step3vConfig,
                                             StepAudioQwen2Config,
53
                                             Telechat2Config, UltravoxConfig)
54
# yapf: enable
55
from vllm.transformers_utils.utils import check_gguf_file
56
from vllm.utils import resolve_obj_by_qualname
57

58
if envs.VLLM_USE_MODELSCOPE:
59
60
61
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
62

63
64
MISTRAL_CONFIG_NAME = "params.json"

65
66
logger = init_logger(__name__)

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

def _get_hf_token() -> Optional[str]:
    """
    Get the HuggingFace token from environment variable.

    Returns None if the token is not set, is an empty string, 
    or contains only whitespace.
    This follows the same pattern as huggingface_hub library which
    treats empty string tokens as None to avoid authentication errors.
    """
    token = os.getenv('HF_TOKEN')
    if token and token.strip():
        return token
    return None


83
_CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = {
84
85
86
    "mllama": MllamaConfig
}

zhuwenwen's avatar
zhuwenwen committed
87
88
89
90
91
92
93
94
95
96
97
98
99
100
_CUSTOM_CONFIG_STEP = {
    "step1": Step1Config,
    "step2": Step2Config,
    "step2_mini": Step2MiniConfig,
    "mmgpt_step1": MMGPTStep1Config,
    "mmgpt_step1_v2": MMGPTStep1ConfigV2,
    #"mmgpt_qwen2": MMGPTQwen2Config,
    #"mmgpt_qwen2_v2": MMGPTQwen2ConfigV2,
    "step1o": Step1oConfig,
    "step1_audio": Step1AudioConfig,
    "step_audio_qwen2": StepAudioQwen2Config,
    "step3v": Step3vConfig,
}

101
_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
GoHomeToMacDonal's avatar
GoHomeToMacDonal committed
102
    "chatglm": ChatGLMConfig,
103
    "cohere2": Cohere2Config,
104
    "dbrx": DbrxConfig,
105
    "deepseek_vl_v2": DeepseekVLV2Config,
106
    "kimi_vl": KimiVLConfig,
107
    "mpt": MPTConfig,
Zhuohan Li's avatar
Zhuohan Li committed
108
109
    "RefinedWeb": RWConfig,  # For tiiuae/falcon-40b(-instruct)
    "RefinedWebModel": RWConfig,  # For tiiuae/falcon-7b(-instruct)
110
    "jais": JAISConfig,
111
    "mlp_speculator": MLPSpeculatorConfig,
112
    "medusa": MedusaConfig,
113
    "eagle": EAGLEConfig,
114
    "exaone": ExaoneConfig,
115
116
    "minimax_text_01": MiniMaxText01Config,
    "minimax_vl_01": MiniMaxVL01Config,
117
    "nemotron": NemotronConfig,
118
    "NVLM_D": NVLM_D_Config,
119
    "ovis": OvisConfig,
120
    "solar": SolarConfig,
121
    "skywork_chat": SkyworkR1VChatConfig,
122
    "telechat": Telechat2Config,
123
    "ultravox": UltravoxConfig,
zhuwenwen's avatar
zhuwenwen committed
124
125
    "step3_vl": Step3VLConfig,
    "step3_text": Step3TextConfig,
zhuwenwen's avatar
zhuwenwen committed
126
127
    **_CONFIG_REGISTRY_OVERRIDE_HF,
    **_CUSTOM_CONFIG_STEP
128
129
}

130
131
132
133
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
    "llm_config": "text_config",
}

134

135
136
137
138
139
140
class ConfigFormat(str, enum.Enum):
    AUTO = "auto"
    HF = "hf"
    MISTRAL = "mistral"


141
142
143
144
145
146
147
148
149
_R = TypeVar("_R")


def with_retry(
    func: Callable[[], _R],
    log_msg: str,
    max_retries: int = 2,
    retry_delay: int = 2,
) -> _R:
150
151
152
153
154
155
156
157
158
159
160
161
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                logger.error("%s: %s", log_msg, e)
                raise
            logger.error("%s: %s, retrying %d of %d", log_msg, e, attempt + 1,
                         max_retries)
            time.sleep(retry_delay)
            retry_delay *= 2

162
163
    raise AssertionError("Should not be reached")

164
165
166
167
168
169
170
171
172
173
174

# @cache doesn't cache exceptions
@cache
def list_repo_files(
    repo_id: str,
    *,
    revision: Optional[str] = None,
    repo_type: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> list[str]:

175
176
177
178
179
180
181
182
    def lookup_files() -> list[str]:
        # directly list files if model is local
        if (local_path := Path(repo_id)).exists():
            return [
                str(file.relative_to(local_path))
                for file in local_path.rglob('*') if file.is_file()
            ]
        # if model is remote, use hf_hub api to list files
183
        try:
184
            if envs.VLLM_USE_MODELSCOPE:
185
186
187
188
                from vllm.transformers_utils.utils import (
                    modelscope_list_repo_files)
                return modelscope_list_repo_files(repo_id,
                                                  revision=revision,
189
190
191
                                                  token=os.getenv(
                                                      "MODELSCOPE_API_TOKEN",
                                                      None))
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
            return hf_list_repo_files(repo_id,
                                      revision=revision,
                                      repo_type=repo_type,
                                      token=token)
        except huggingface_hub.errors.OfflineModeIsEnabled:
            # Don't raise in offline mode,
            # all we know is that we don't have this
            # file cached.
            return []

    return with_retry(lookup_files, "Error retrieving file list")


def file_exists(
    repo_id: str,
    file_name: str,
    *,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> bool:
    file_list = list_repo_files(repo_id,
                                repo_type=repo_type,
                                revision=revision,
                                token=token)
    return file_name in file_list


# In offline mode the result can be a false negative
221
222
def file_or_path_exists(model: Union[str, Path], config_name: str,
                        revision: Optional[str]) -> bool:
223
224
    if (local_path := Path(model)).exists():
        return (local_path / config_name).is_file()
225

Joe Runde's avatar
Joe Runde committed
226
227
228
229
230
231
232
233
234
235
    # Offline mode support: Check if config file is cached already
    cached_filepath = try_to_load_from_cache(repo_id=model,
                                             filename=config_name,
                                             revision=revision)
    if isinstance(cached_filepath, str):
        # The config file exists in cache- we can continue trying to load
        return True

    # NB: file_exists will only check for the existence of the config file on
    # hf_hub. This will fail in offline mode.
236
237

    # Call HF to check if the file exists
238
239
240
    return file_exists(str(model),
                       config_name,
                       revision=revision,
241
                       token=_get_hf_token())
242
243


244
245
246
247
248
249
250
251
252
253
254
def patch_rope_scaling(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        patch_rope_scaling(text_config)

    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is not None:
        patch_rope_scaling_dict(rope_scaling)


255
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
256
257
258
259
260
261
262
263
264
    if "rope_type" in rope_scaling and "type" in rope_scaling:
        rope_type = rope_scaling["rope_type"]
        rope_type_legacy = rope_scaling["type"]
        if rope_type != rope_type_legacy:
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
                "You should only specify one of them.")

265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
    if "rope_type" not in rope_scaling and "type" in rope_scaling:
        rope_scaling["rope_type"] = rope_scaling["type"]
        logger.info("Replacing legacy 'type' key with 'rope_type'")

    if "rope_type" not in rope_scaling:
        raise ValueError("rope_scaling should have a 'rope_type' key")

    if rope_scaling["rope_type"] == "su":
        rope_scaling["rope_type"] = "longrope"
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
    elif rope_scaling["rope_type"] == "mrope":
        assert "mrope_section" in rope_scaling
        rope_scaling["rope_type"] = "default"
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


281
def _uses_mrope(config: PretrainedConfig) -> bool:
282
283
284
285
286
287
288
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return False

    return "mrope_section" in rope_scaling


289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
    return _uses_mrope(config) or thinker_uses_mrope(config)


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


307
308
309
310
311
312
313
314
315
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        return is_encoder_decoder(text_config)

    return getattr(config, "is_encoder_decoder", False)


316
317
318
319
320
321
322
323
324
325
326
327
def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
    """Remap config attributes to match the expected names."""
    for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
        if hasattr(config, old_attr):
            if not hasattr(config, new_attr):
                config.update({new_attr: getattr(config, old_attr)})
            delattr(config, old_attr)
            logger.debug("Remapped config attribute '%s' to '%s'", old_attr,
                         new_attr)
    return config


328
329
330
331
332
def get_config(
    model: Union[str, Path],
    trust_remote_code: bool,
    revision: Optional[str] = None,
    code_revision: Optional[str] = None,
333
    config_format: ConfigFormat = ConfigFormat.AUTO,
334
335
336
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
337

338
    is_gguf = check_gguf_file(model)
339
340
341
342
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        model = Path(model).parent

343
    if config_format == ConfigFormat.AUTO:
344
345
346
347
348
349
350
351
        try:
            if is_gguf or file_or_path_exists(
                    model, HF_CONFIG_NAME, revision=revision):
                config_format = ConfigFormat.HF
            elif file_or_path_exists(model,
                                     MISTRAL_CONFIG_NAME,
                                     revision=revision):
                config_format = ConfigFormat.MISTRAL
352
353
354
355
356
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
                    "Ensure your model has either config.json (HF format) "
                    "or params.json (Mistral format).")
357
358
359
360
361
362
363
364
365
366
367

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
368
369
370
371
                "'params.json'.\n"
                "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
                "   Loading GGUF from a remote repo directly is not yet "
                "supported.\n").format(model=model)
372
373

            raise ValueError(error_message) from e
374
375
376

    if config_format == ConfigFormat.HF:
        config_dict, _ = PretrainedConfig.get_config_dict(
377
378
379
            model,
            revision=revision,
            code_revision=code_revision,
380
            token=_get_hf_token(),
381
382
            **kwargs,
        )
383
384
385
386
387

        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
388
389
390
391
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
392
                token=_get_hf_token(),
393
394
                **kwargs,
            )
395
396
397
398
399
400
401
        else:
            try:
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
402
                    token=_get_hf_token(),
403
404
405
                    # some old custom model's config needs
                    # `has_no_defaults_at_init=True` to work.
                    has_no_defaults_at_init=trust_remote_code,
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
                    **kwargs,
                )
            except ValueError as e:
                if (not trust_remote_code
                        and "requires you to execute the configuration file"
                        in str(e)):
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
                        "`--trust-remote-code` flag in the CLI.")
                    raise RuntimeError(err_msg) from e
                else:
                    raise e
421
        config = _maybe_remap_hf_config_attrs(config)
422
423

    elif config_format == ConfigFormat.MISTRAL:
424
        config = load_params_config(model, revision, **kwargs)
425
    else:
426
427
428
429
430
431
432
433
        supported_formats = [
            fmt.value for fmt in ConfigFormat if fmt != ConfigFormat.AUTO
        ]
        raise ValueError(
            f"Unsupported config format: {config_format}. "
            f"Supported formats are: {', '.join(supported_formats)}. "
            f"Ensure your model uses one of these configuration formats "
            f"or specify the correct format explicitly.")
434
435
436
437
438
439
440
441
442

    # Special architecture mapping check for GGUF models
    if is_gguf:
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
            raise RuntimeError(
                f"Can't get gguf config for {config.model_type}.")
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

443
444
    patch_rope_scaling(config)

445
446
447
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

448
    return config
449
450


451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
def try_get_local_file(model: Union[str, Path],
                       file_name: str,
                       revision: Optional[str] = 'main') -> Optional[Path]:
    file_path = Path(model) / file_name
    if file_path.is_file():
        return file_path
    else:
        try:
            cached_filepath = try_to_load_from_cache(repo_id=model,
                                                     filename=file_name,
                                                     revision=revision)
            if isinstance(cached_filepath, str):
                return Path(cached_filepath)
        except HFValidationError:
            ...
    return None


469
470
def get_hf_file_to_dict(file_name: str,
                        model: Union[str, Path],
471
                        revision: Optional[str] = 'main'):
472
    """
473
    Downloads a file from the Hugging Face Hub and returns
474
475
476
477
478
    its contents as a dictionary.

    Parameters:
    - file_name (str): The name of the file to download.
    - model (str): The name of the model on the Hugging Face Hub.
479
    - revision (str): The specific version of the model.
480
481

    Returns:
482
    - config_dict (dict): A dictionary containing
483
484
485
    the contents of the downloaded file.
    """

486
487
488
489
    file_path = try_get_local_file(model=model,
                                   file_name=file_name,
                                   revision=revision)

490
    if file_path is None:
491
492
        try:
            hf_hub_file = hf_hub_download(model, file_name, revision=revision)
493
494
        except huggingface_hub.errors.OfflineModeIsEnabled:
            return None
495
496
497
498
499
500
501
502
503
504
505
506
507
508
        except (RepositoryNotFoundError, RevisionNotFoundError,
                EntryNotFoundError, LocalEntryNotFoundError) as e:
            logger.debug("File or repository not found in hf_hub_download", e)
            return None
        except HfHubHTTPError as e:
            logger.warning(
                "Cannot connect to Hugging Face Hub. Skipping file "
                "download for '%s':",
                file_name,
                exc_info=e)
            return None
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
509
510
        with open(file_path) as file:
            return json.load(file)
511

512
513
514
    return None


515
@cache
516
def get_pooling_config(model: str, revision: Optional[str] = 'main'):
517
    """
518
519
520
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
521
522
523

    Args:
        model (str): The name of the Hugging Face model.
524
        revision (str, optional): The specific version
525
526
527
        of the model to use. Defaults to 'main'.

    Returns:
528
        dict: A dictionary containing the pooling
529
530
531
532
        type and whether normalization is used.
    """

    modules_file_name = "modules.json"
533
534
535
536
537
538

    modules_dict = None
    if file_or_path_exists(model=model,
                           config_name=modules_file_name,
                           revision=revision):
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
539
540
541
542

    if modules_dict is None:
        return None

543
544
    logger.info("Found sentence-transformers modules configuration.")

545
546
547
548
549
550
551
552
553
554
555
    pooling = next((item for item in modules_dict
                    if item["type"] == "sentence_transformers.models.Pooling"),
                   None)
    normalize = bool(
        next((item for item in modules_dict
              if item["type"] == "sentence_transformers.models.Normalize"),
             False))

    if pooling:

        pooling_file_name = "{}/config.json".format(pooling["path"])
556
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
557
558
559
560
561
562
        pooling_type_name = next(
            (item for item, val in pooling_dict.items() if val is True), None)

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

563
        logger.info("Found pooling configuration.")
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

    supported_pooling_types = ['LAST', 'ALL', 'CLS', 'STEP', 'MEAN']
    pooling_type_name = pooling_name.upper()

    try:
        if pooling_type_name in supported_pooling_types:
            return pooling_type_name
    except NotImplementedError as e:
        logger.debug("Pooling type not supported", e)
        return None
    return None


591
@cache
592
def get_sentence_transformer_tokenizer_config(model: str,
593
594
                                              revision: Optional[str] = 'main'
                                              ):
595
    """
596
    Returns the tokenization configuration dictionary for a
597
598
599
    given Sentence Transformer BERT model.

    Parameters:
600
    - model (str): The name of the Sentence Transformer
601
602
603
604
605
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
606
    - dict: A dictionary containing the configuration parameters
607
608
    for the Sentence Transformer BERT model.
    """
609
610
611
612
613
614
615
616
617
618
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
619
620
621
622
623
624

    for config_file in sentence_transformer_config_files:
        if try_get_local_file(model=model,
                              file_name=config_file,
                              revision=revision) is not None:
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
625
626
            if encoder_dict:
                break
627

628
    if not encoder_dict and not model.startswith("/"):
629
630
631
632
        try:
            # If model is on HuggingfaceHub, get the repo files
            repo_files = list_repo_files(model,
                                         revision=revision,
633
                                         token=_get_hf_token())
634
        except Exception:
635
636
637
638
639
640
641
642
643
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
                encoder_dict = get_hf_file_to_dict(config_name, model,
                                                   revision)
                if encoder_dict:
                    break

644
645
646
    if not encoder_dict:
        return None

647
648
    logger.info("Found sentence-transformers tokenize configuration.")

649
650
651
652
653
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


654
def maybe_register_config_serialize_by_value() -> None:
655
656
    """Try to register HF model configuration class to serialize by value

657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
        If trust_remote_code is set, and the model's config file specifies an
        `AutoConfig` class, then the config class is typically an instance of
        a custom class imported from the HF modules cache.

        Examples:

        >>> from transformers import AutoConfig
        >>> klass = AutoConfig.from_pretrained('meta-llama/Meta-Llama-3-8B', trust_remote_code=True)
        >>> klass.__class__ # transformers.models.llama.configuration_llama.LlamaConfig
        >>> import transformers_modules # error, not initialized
        >>> klass = AutoConfig.from_pretrained('deepseek-ai/DeepSeek-V2.5', trust_remote_code=True)
        >>> import transformers_modules # success, initialized
        >>> klass.__class__ # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

        In the DeepSeek example, the config class is an instance of a custom
        class that is not serializable by default. This class will not be
        importable in spawned workers, and won't exist at all on
        other nodes, which breaks serialization of the config.
675
676
677
678

        In this function we tell the cloudpickle serialization library to pass
        instances of these generated classes by value instead of by reference,
        i.e. the class definition is serialized along with its data so that the
679
        class module does not need to be importable on the receiving end.
680
681

        See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
682
    """ # noqa
683
684
    try:
        import transformers_modules
685
        transformers_modules_available = True
686
    except ImportError:
687
        transformers_modules_available = False
688
689
690
691
692

    try:
        import multiprocessing
        import pickle

693
694
        import cloudpickle

695
        from vllm.config import VllmConfig
696

697
698
699
        # Register multiprocessing reducers to handle cross-process
        # serialization of VllmConfig objects that may contain custom configs
        # from transformers_modules
700
701
        def _reduce_config(config: VllmConfig):
            return (pickle.loads, (cloudpickle.dumps(config), ))
702

703
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
704

705
706
707
708
709
710
711
712
713
        # Register transformers_modules with cloudpickle if available
        if transformers_modules_available:
            cloudpickle.register_pickle_by_value(transformers_modules)

            # ray vendors its own version of cloudpickle
            from vllm.executor.ray_utils import ray
            if ray:
                ray.cloudpickle.register_pickle_by_value(transformers_modules)

714
715
716
717
718
719
720
721
722
    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
            exc_info=e)


723
def load_params_config(model: Union[str, Path], revision: Optional[str],
724
                       **kwargs) -> PretrainedConfig:
725
726
727
728
729
    # This function loads a params.json config which
    # should be used when loading models in mistral format

    config_file_name = "params.json"

730
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
731
732
733
734
735
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
            f"and if the config file exists.")
736
    assert isinstance(config_dict, dict)
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752

    config_mapping = {
        "dim": "hidden_size",
        "norm_eps": "rms_norm_eps",
        "n_kv_heads": "num_key_value_heads",
        "n_layers": "num_hidden_layers",
        "n_heads": "num_attention_heads",
        "hidden_dim": "intermediate_size",
    }

    def recurse_elems(elem: Any):
        if isinstance(elem, dict):
            config_dict = {}
            for key, value in elem.items():
                key = config_mapping.get(key, key)
                config_dict[key] = recurse_elems(value)
753
754

            return config_dict
755
756
757
758
759
760
761
        else:
            return elem

    config_dict["model_type"] = config_dict.get("model_type", "transformer")
    config_dict["hidden_act"] = config_dict.get("activation", "silu")
    config_dict["tie_word_embeddings"] = config_dict.get(
        "tie_embeddings", False)
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779

    if config_dict.get("max_position_embeddings") is None:
        max_position_embeddings = 128_000
        try:
            trust_remote_code_val = kwargs.get("trust_remote_code", False)
            hf_config = get_config(model=model,
                                   trust_remote_code=trust_remote_code_val,
                                   revision=revision,
                                   config_format=ConfigFormat.HF)
            if hf_value := hf_config.get_text_config().max_position_embeddings:
                max_position_embeddings = hf_value
        except Exception as e:
            logger.warning(
                "The params.json file is missing 'max_position_embeddings'"
                " and could not get a value from the HF config."
                " Defaulting to 128000",
                exc_info=e)
        config_dict["max_position_embeddings"] = max_position_embeddings
780

781
782
783
784
785
786
787
788
    if config_dict.get("quantization") is not None:
        quantization = config_dict.get("quantization", {})
        if quantization.get("qformat_weight") == "fp8_e4m3":
            # This maps to the FP8 static per-tensor quantization scheme
            quantization_config = {
                "quant_method": "fp8",
                "activation_scheme": "static"
            }
789
790
791
        elif quantization.get("quant_method") == "compressed-tensors":
            # Pass through the quantization config to compressed-tensors
            quantization_config = quantization
792
793
794
795
796
797
798
799
800
801
        else:
            raise ValueError(
                f"Found unknown quantization='{quantization}' in config")

        config_dict["quantization_config"] = quantization_config

    config_type: Literal["text",
                         "multimodal"] = "multimodal" if config_dict.get(
                             "vision_encoder") is not None else "text"

Patrick von Platen's avatar
Patrick von Platen committed
802
803
804
805
806
    if config_dict.get("moe") is not None:
        config_dict["architectures"] = ["MixtralForCausalLM"]
    else:
        config_dict["architectures"] = ["MistralForCausalLM"]

807
    if config_type == "multimodal":
Patrick von Platen's avatar
Patrick von Platen committed
808
        multimodal_config = config_dict.pop("vision_encoder")
809
        quantization_config = config_dict.get("quantization_config", {})
810

Patrick von Platen's avatar
Patrick von Platen committed
811
812
813
814
815
816
        config_dict = {
            "text_config": config_dict,
            "vision_config": multimodal_config
        }
        config_dict["architectures"] = ["PixtralForConditionalGeneration"]
        config_dict["model_type"] = "pixtral"
817
818
        if quantization_config:
            config_dict["quantization_config"] = quantization_config
Patrick von Platen's avatar
Patrick von Platen committed
819

820
821
    config_dict.update(kwargs)

822
823
824
825
826
827
828
829
830
831
    config_dict = recurse_elems(config_dict)

    # transform to HF config format
    if config_type == "multimodal":
        config_dict["text_config"] = PretrainedConfig(
            **config_dict["text_config"])
        config_dict["vision_config"] = PretrainedConfig(
            **config_dict["vision_config"])

    return PretrainedConfig(**config_dict)
832
833


834
835
def get_hf_image_processor_config(
    model: Union[str, Path],
836
    hf_token: Optional[Union[bool, str]] = None,
837
838
    revision: Optional[str] = None,
    **kwargs,
839
) -> dict[str, Any]:
840
    # ModelScope does not provide an interface for image_processor
841
    if envs.VLLM_USE_MODELSCOPE:
842
        return dict()
843
    # Separate model folder from file path for GGUF models
844
    if check_gguf_file(model):
845
        model = Path(model).parent
846
847
848
849
    return get_image_processor_config(model,
                                      token=hf_token,
                                      revision=revision,
                                      **kwargs)
850
851


852
853
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
854
    No op for pure text models.
855
    """
856
857
    # This block should be unnecessary after https://github.com/huggingface/transformers/pull/37517
    if hasattr(config, "thinker_config"):
858
859
860
861
        # TODO(suyang.fy): Refactor code.
        #  For Qwen2.5-Omni, change hf_text_config to
        #  thinker_config.text_config.
        return config.thinker_config.text_config
862
863
864
865
866
867
868
869
870
871

    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
    revision: Optional[str] = None,
) -> Optional[GenerationConfig]:
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
894
895


896
897
898
def get_classification_activation_function(config: PretrainedConfig):
    return nn.Sigmoid() if config.num_labels == 1 else nn.Softmax()

899

900
def get_cross_encoder_activation_function(config: PretrainedConfig):
901
    function_name: Optional[str] = None
902
903
    if (hasattr(config, "sentence_transformers")
            and "activation_fn" in config.sentence_transformers):
904
905
906
        function_name = config.sentence_transformers["activation_fn"]
    elif (hasattr(config, "sbert_ce_default_activation_function")
          and config.sbert_ce_default_activation_function is not None):
907
        function_name = config.sbert_ce_default_activation_function
908
909

    if function_name is not None:
910
911
912
        assert function_name.startswith("torch.nn.modules."), (
            "Loading of activation functions is restricted to "
            "torch.nn.modules for security reasons")
913
        return resolve_obj_by_qualname(function_name)()
914
915

    return nn.Sigmoid() if config.num_labels == 1 else nn.Identity()
916
917
918
919
920
921
922
923
924
925
926


def try_get_safetensors_metadata(
    model: str,
    *,
    revision: Optional[str] = None,
):
    get_safetensors_metadata_partial = partial(
        get_safetensors_metadata,
        model,
        revision=revision,
927
        token=_get_hf_token(),
928
929
930
931
932
933
934
    )

    try:
        return with_retry(get_safetensors_metadata_partial,
                          "Error retrieving safetensors")
    except Exception:
        return None
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949


def try_get_tokenizer_config(
    pretrained_model_name_or_path: Union[str, os.PathLike],
    trust_remote_code: bool,
    revision: Optional[str] = None,
) -> Optional[dict[str, Any]]:
    try:
        return get_tokenizer_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            revision=revision,
        )
    except Exception:
        return None