config.py 30.7 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
4
import enum
import json
5
import os
6
import time
7
from functools import cache
8
from pathlib import Path
9
from typing import Any, Callable, Literal, Optional, Union
Jasmond L's avatar
Jasmond L committed
10

Joe Runde's avatar
Joe Runde committed
11
import huggingface_hub
12
13
14
from huggingface_hub import hf_hub_download
from huggingface_hub import list_repo_files as hf_list_repo_files
from huggingface_hub import try_to_load_from_cache
15
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
16
                                   HFValidationError, LocalEntryNotFoundError,
17
18
                                   RepositoryNotFoundError,
                                   RevisionNotFoundError)
19
from torch import nn
20
from transformers import GenerationConfig, PretrainedConfig
21
22
from transformers.models.auto.image_processing_auto import (
    get_image_processor_config)
23
24
from transformers.models.auto.modeling_auto import (
    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
25
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26

27
from vllm.envs import VLLM_USE_MODELSCOPE
28
from vllm.logger import init_logger
29
30
# yapf conflicts with isort for this block
# yapf: disable
31
32
33
34
from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
                                             DbrxConfig, DeepseekVLV2Config,
                                             EAGLEConfig, ExaoneConfig,
                                             H2OVLChatConfig,
35
                                             InternVLChatConfig, JAISConfig,
36
                                             KimiVLConfig, MedusaConfig,
37
38
39
40
                                             MiniMaxText01Config,
                                             MiniMaxVL01Config, MllamaConfig,
                                             MLPSpeculatorConfig, MPTConfig,
                                             NemotronConfig, NVLM_D_Config,
41
42
43
                                             OvisConfig, RWConfig,
                                             SkyworkR1VChatConfig, SolarConfig,
                                             Telechat2Config, UltravoxConfig)
44
# yapf: enable
45
from vllm.transformers_utils.utils import check_gguf_file
46
from vllm.utils import resolve_obj_by_qualname
47
48
49
50
51

if VLLM_USE_MODELSCOPE:
    from modelscope import AutoConfig
else:
    from transformers import AutoConfig
52

53
MISTRAL_CONFIG_NAME = "params.json"
54
HF_TOKEN = os.getenv('HF_TOKEN', None)
55

56
57
logger = init_logger(__name__)

58
_CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = {
59
60
61
    "mllama": MllamaConfig
}

62
_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
GoHomeToMacDonal's avatar
GoHomeToMacDonal committed
63
    "chatglm": ChatGLMConfig,
64
    "cohere2": Cohere2Config,
65
    "dbrx": DbrxConfig,
66
    "deepseek_vl_v2": DeepseekVLV2Config,
67
    "kimi_vl": KimiVLConfig,
68
    "mpt": MPTConfig,
Zhuohan Li's avatar
Zhuohan Li committed
69
70
    "RefinedWeb": RWConfig,  # For tiiuae/falcon-40b(-instruct)
    "RefinedWebModel": RWConfig,  # For tiiuae/falcon-7b(-instruct)
71
    "jais": JAISConfig,
72
    "mlp_speculator": MLPSpeculatorConfig,
73
    "medusa": MedusaConfig,
74
    "eagle": EAGLEConfig,
75
    "exaone": ExaoneConfig,
76
    "h2ovl_chat": H2OVLChatConfig,
77
    "internvl_chat": InternVLChatConfig,
78
79
    "minimax_text_01": MiniMaxText01Config,
    "minimax_vl_01": MiniMaxVL01Config,
80
    "nemotron": NemotronConfig,
81
    "NVLM_D": NVLM_D_Config,
82
    "ovis": OvisConfig,
83
    "solar": SolarConfig,
84
    "skywork_chat": SkyworkR1VChatConfig,
85
    "telechat": Telechat2Config,
86
    "ultravox": UltravoxConfig,
87
    **_CONFIG_REGISTRY_OVERRIDE_HF
88
89
90
}


91
92
93
94
95
96
class ConfigFormat(str, enum.Enum):
    AUTO = "auto"
    HF = "hf"
    MISTRAL = "mistral"


97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def with_retry(func: Callable[[], Any],
               log_msg: str,
               max_retries: int = 2,
               retry_delay: int = 2):
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                logger.error("%s: %s", log_msg, e)
                raise
            logger.error("%s: %s, retrying %d of %d", log_msg, e, attempt + 1,
                         max_retries)
            time.sleep(retry_delay)
            retry_delay *= 2


# @cache doesn't cache exceptions
@cache
def list_repo_files(
    repo_id: str,
    *,
    revision: Optional[str] = None,
    repo_type: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> list[str]:

124
125
126
127
128
129
130
131
    def lookup_files() -> list[str]:
        # directly list files if model is local
        if (local_path := Path(repo_id)).exists():
            return [
                str(file.relative_to(local_path))
                for file in local_path.rglob('*') if file.is_file()
            ]
        # if model is remote, use hf_hub api to list files
132
        try:
133
134
135
136
137
138
            if VLLM_USE_MODELSCOPE:
                from vllm.transformers_utils.utils import (
                    modelscope_list_repo_files)
                return modelscope_list_repo_files(repo_id,
                                                  revision=revision,
                                                  token=token)
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
            return hf_list_repo_files(repo_id,
                                      revision=revision,
                                      repo_type=repo_type,
                                      token=token)
        except huggingface_hub.errors.OfflineModeIsEnabled:
            # Don't raise in offline mode,
            # all we know is that we don't have this
            # file cached.
            return []

    return with_retry(lookup_files, "Error retrieving file list")


def file_exists(
    repo_id: str,
    file_name: str,
    *,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
    token: Union[str, bool, None] = None,
) -> bool:
    file_list = list_repo_files(repo_id,
                                repo_type=repo_type,
                                revision=revision,
                                token=token)
    return file_name in file_list


# In offline mode the result can be a false negative
168
169
def file_or_path_exists(model: Union[str, Path], config_name: str,
                        revision: Optional[str]) -> bool:
170
171
    if (local_path := Path(model)).exists():
        return (local_path / config_name).is_file()
172

Joe Runde's avatar
Joe Runde committed
173
174
175
176
177
178
179
180
181
182
    # Offline mode support: Check if config file is cached already
    cached_filepath = try_to_load_from_cache(repo_id=model,
                                             filename=config_name,
                                             revision=revision)
    if isinstance(cached_filepath, str):
        # The config file exists in cache- we can continue trying to load
        return True

    # NB: file_exists will only check for the existence of the config file on
    # hf_hub. This will fail in offline mode.
183
184

    # Call HF to check if the file exists
185
186
187
188
    return file_exists(str(model),
                       config_name,
                       revision=revision,
                       token=HF_TOKEN)
189
190


191
192
193
194
195
196
197
198
199
200
201
def patch_rope_scaling(config: PretrainedConfig) -> None:
    """Provide backwards compatibility for RoPE."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        patch_rope_scaling(text_config)

    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is not None:
        patch_rope_scaling_dict(rope_scaling)


202
def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
203
204
205
206
207
208
209
210
211
    if "rope_type" in rope_scaling and "type" in rope_scaling:
        rope_type = rope_scaling["rope_type"]
        rope_type_legacy = rope_scaling["type"]
        if rope_type != rope_type_legacy:
            raise ValueError(
                f"Found conflicts between 'rope_type={rope_type}' (modern "
                f"field) and 'type={rope_type_legacy}' (legacy field). "
                "You should only specify one of them.")

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    if "rope_type" not in rope_scaling and "type" in rope_scaling:
        rope_scaling["rope_type"] = rope_scaling["type"]
        logger.info("Replacing legacy 'type' key with 'rope_type'")

    if "rope_type" not in rope_scaling:
        raise ValueError("rope_scaling should have a 'rope_type' key")

    if rope_scaling["rope_type"] == "su":
        rope_scaling["rope_type"] = "longrope"
        logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
    elif rope_scaling["rope_type"] == "mrope":
        assert "mrope_section" in rope_scaling
        rope_scaling["rope_type"] = "default"
        logger.warning("Replacing legacy rope_type 'mrope' with 'default'")


228
def _uses_mrope(config: PretrainedConfig) -> bool:
229
230
231
232
233
234
235
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling is None:
        return False

    return "mrope_section" in rope_scaling


236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
def uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model with this config uses M-ROPE."""
    return _uses_mrope(config) or thinker_uses_mrope(config)


def thinker_uses_mrope(config: PretrainedConfig) -> bool:
    """Detect if the model contains a thinker config and it uses M-ROPE."""
    thinker_config = getattr(config, "thinker_config", None)
    if thinker_config is None:
        return False

    thinker_text_config = getattr(thinker_config, "text_config", None)
    if thinker_text_config is None:
        return False

    return uses_mrope(thinker_text_config)


254
255
256
257
258
259
260
261
262
def is_encoder_decoder(config: PretrainedConfig) -> bool:
    """Detect if the model with this config is used as an encoder/decoder."""
    text_config = getattr(config, "text_config", None)
    if text_config is not None:
        return is_encoder_decoder(text_config)

    return getattr(config, "is_encoder_decoder", False)


263
264
265
266
267
def get_config(
    model: Union[str, Path],
    trust_remote_code: bool,
    revision: Optional[str] = None,
    code_revision: Optional[str] = None,
268
    config_format: ConfigFormat = ConfigFormat.AUTO,
269
270
271
    **kwargs,
) -> PretrainedConfig:
    # Separate model folder from file path for GGUF models
272

273
    is_gguf = check_gguf_file(model)
274
275
276
277
    if is_gguf:
        kwargs["gguf_file"] = Path(model).name
        model = Path(model).parent

278
    if config_format == ConfigFormat.AUTO:
279
280
281
282
283
284
285
286
        try:
            if is_gguf or file_or_path_exists(
                    model, HF_CONFIG_NAME, revision=revision):
                config_format = ConfigFormat.HF
            elif file_or_path_exists(model,
                                     MISTRAL_CONFIG_NAME,
                                     revision=revision):
                config_format = ConfigFormat.MISTRAL
287
288
289
290
291
            else:
                raise ValueError(
                    "Could not detect config format for no config file found. "
                    "Ensure your model has either config.json (HF format) "
                    "or params.json (Mistral format).")
292
293
294
295
296
297
298
299
300
301
302

        except Exception as e:
            error_message = (
                "Invalid repository ID or local directory specified:"
                " '{model}'.\nPlease verify the following requirements:\n"
                "1. Provide a valid Hugging Face repository ID.\n"
                "2. Specify a local directory that contains a recognized "
                "configuration file.\n"
                "   - For Hugging Face models: ensure the presence of a "
                "'config.json'.\n"
                "   - For Mistral models: ensure the presence of a "
303
304
305
306
                "'params.json'.\n"
                "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
                "   Loading GGUF from a remote repo directly is not yet "
                "supported.\n").format(model=model)
307
308

            raise ValueError(error_message) from e
309
310
311

    if config_format == ConfigFormat.HF:
        config_dict, _ = PretrainedConfig.get_config_dict(
312
313
314
            model,
            revision=revision,
            code_revision=code_revision,
315
            token=HF_TOKEN,
316
317
            **kwargs,
        )
318
319
320
321
322

        # Use custom model class if it's in our registry
        model_type = config_dict.get("model_type")
        if model_type in _CONFIG_REGISTRY:
            config_class = _CONFIG_REGISTRY[model_type]
323
324
325
326
            config = config_class.from_pretrained(
                model,
                revision=revision,
                code_revision=code_revision,
327
                token=HF_TOKEN,
328
329
                **kwargs,
            )
330
331
332
333
334
335
336
        else:
            try:
                config = AutoConfig.from_pretrained(
                    model,
                    trust_remote_code=trust_remote_code,
                    revision=revision,
                    code_revision=code_revision,
337
                    token=HF_TOKEN,
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
                    **kwargs,
                )
            except ValueError as e:
                if (not trust_remote_code
                        and "requires you to execute the configuration file"
                        in str(e)):
                    err_msg = (
                        "Failed to load the model config. If the model "
                        "is a custom model not yet available in the "
                        "HuggingFace transformers library, consider setting "
                        "`trust_remote_code=True` in LLM or using the "
                        "`--trust-remote-code` flag in the CLI.")
                    raise RuntimeError(err_msg) from e
                else:
                    raise e

    elif config_format == ConfigFormat.MISTRAL:
355
        config = load_params_config(model, revision, token=HF_TOKEN, **kwargs)
356
    else:
357
358
359
360
361
362
363
364
        supported_formats = [
            fmt.value for fmt in ConfigFormat if fmt != ConfigFormat.AUTO
        ]
        raise ValueError(
            f"Unsupported config format: {config_format}. "
            f"Supported formats are: {', '.join(supported_formats)}. "
            f"Ensure your model uses one of these configuration formats "
            f"or specify the correct format explicitly.")
365
366
367
368
369
370
371
372
373

    # Special architecture mapping check for GGUF models
    if is_gguf:
        if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
            raise RuntimeError(
                f"Can't get gguf config for {config.model_type}.")
        model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
        config.update({"architectures": [model_type]})

374
375
    patch_rope_scaling(config)

376
377
378
    if trust_remote_code:
        maybe_register_config_serialize_by_value()

379
    return config
380
381


382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
def try_get_local_file(model: Union[str, Path],
                       file_name: str,
                       revision: Optional[str] = 'main') -> Optional[Path]:
    file_path = Path(model) / file_name
    if file_path.is_file():
        return file_path
    else:
        try:
            cached_filepath = try_to_load_from_cache(repo_id=model,
                                                     filename=file_name,
                                                     revision=revision)
            if isinstance(cached_filepath, str):
                return Path(cached_filepath)
        except HFValidationError:
            ...
    return None


400
401
def get_hf_file_to_dict(file_name: str,
                        model: Union[str, Path],
402
                        revision: Optional[str] = 'main'):
403
    """
404
    Downloads a file from the Hugging Face Hub and returns
405
406
407
408
409
    its contents as a dictionary.

    Parameters:
    - file_name (str): The name of the file to download.
    - model (str): The name of the model on the Hugging Face Hub.
410
    - revision (str): The specific version of the model.
411
412

    Returns:
413
    - config_dict (dict): A dictionary containing
414
415
416
    the contents of the downloaded file.
    """

417
418
419
420
    file_path = try_get_local_file(model=model,
                                   file_name=file_name,
                                   revision=revision)

421
    if file_path is None:
422
423
        try:
            hf_hub_file = hf_hub_download(model, file_name, revision=revision)
424
425
        except huggingface_hub.errors.OfflineModeIsEnabled:
            return None
426
427
428
429
430
431
432
433
434
435
436
437
438
439
        except (RepositoryNotFoundError, RevisionNotFoundError,
                EntryNotFoundError, LocalEntryNotFoundError) as e:
            logger.debug("File or repository not found in hf_hub_download", e)
            return None
        except HfHubHTTPError as e:
            logger.warning(
                "Cannot connect to Hugging Face Hub. Skipping file "
                "download for '%s':",
                file_name,
                exc_info=e)
            return None
        file_path = Path(hf_hub_file)

    if file_path is not None and file_path.is_file():
440
441
        with open(file_path) as file:
            return json.load(file)
442

443
444
445
    return None


446
@cache
447
def get_pooling_config(model: str, revision: Optional[str] = 'main'):
448
    """
449
450
451
    This function gets the pooling and normalize
    config from the model - only applies to
    sentence-transformers models.
452
453
454

    Args:
        model (str): The name of the Hugging Face model.
455
        revision (str, optional): The specific version
456
457
458
        of the model to use. Defaults to 'main'.

    Returns:
459
        dict: A dictionary containing the pooling
460
461
462
463
        type and whether normalization is used.
    """

    modules_file_name = "modules.json"
464
465
466
467
468
469

    modules_dict = None
    if file_or_path_exists(model=model,
                           config_name=modules_file_name,
                           revision=revision):
        modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
470
471
472
473

    if modules_dict is None:
        return None

474
475
    logger.info("Found sentence-transformers modules configuration.")

476
477
478
479
480
481
482
483
484
485
486
    pooling = next((item for item in modules_dict
                    if item["type"] == "sentence_transformers.models.Pooling"),
                   None)
    normalize = bool(
        next((item for item in modules_dict
              if item["type"] == "sentence_transformers.models.Normalize"),
             False))

    if pooling:

        pooling_file_name = "{}/config.json".format(pooling["path"])
487
        pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
488
489
490
491
492
493
        pooling_type_name = next(
            (item for item, val in pooling_dict.items() if val is True), None)

        if pooling_type_name is not None:
            pooling_type_name = get_pooling_config_name(pooling_type_name)

494
        logger.info("Found pooling configuration.")
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
        return {"pooling_type": pooling_type_name, "normalize": normalize}

    return None


def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
    if "pooling_mode_" in pooling_name:
        pooling_name = pooling_name.replace("pooling_mode_", "")

    if "_" in pooling_name:
        pooling_name = pooling_name.split("_")[0]

    if "lasttoken" in pooling_name:
        pooling_name = "last"

    supported_pooling_types = ['LAST', 'ALL', 'CLS', 'STEP', 'MEAN']
    pooling_type_name = pooling_name.upper()

    try:
        if pooling_type_name in supported_pooling_types:
            return pooling_type_name
    except NotImplementedError as e:
        logger.debug("Pooling type not supported", e)
        return None
    return None


522
@cache
523
def get_sentence_transformer_tokenizer_config(model: str,
524
525
                                              revision: Optional[str] = 'main'
                                              ):
526
    """
527
    Returns the tokenization configuration dictionary for a
528
529
530
    given Sentence Transformer BERT model.

    Parameters:
531
    - model (str): The name of the Sentence Transformer
532
533
534
535
536
    BERT model.
    - revision (str, optional): The revision of the m
    odel to use. Defaults to 'main'.

    Returns:
537
    - dict: A dictionary containing the configuration parameters
538
539
    for the Sentence Transformer BERT model.
    """
540
541
542
543
544
545
546
547
548
549
    sentence_transformer_config_files = [
        "sentence_bert_config.json",
        "sentence_roberta_config.json",
        "sentence_distilbert_config.json",
        "sentence_camembert_config.json",
        "sentence_albert_config.json",
        "sentence_xlm-roberta_config.json",
        "sentence_xlnet_config.json",
    ]
    encoder_dict = None
550
551
552
553
554
555

    for config_file in sentence_transformer_config_files:
        if try_get_local_file(model=model,
                              file_name=config_file,
                              revision=revision) is not None:
            encoder_dict = get_hf_file_to_dict(config_file, model, revision)
556
557
            if encoder_dict:
                break
558

559
    if not encoder_dict and not model.startswith("/"):
560
561
562
563
564
        try:
            # If model is on HuggingfaceHub, get the repo files
            repo_files = list_repo_files(model,
                                         revision=revision,
                                         token=HF_TOKEN)
565
        except Exception:
566
567
568
569
570
571
572
573
574
            repo_files = []

        for config_name in sentence_transformer_config_files:
            if config_name in repo_files:
                encoder_dict = get_hf_file_to_dict(config_name, model,
                                                   revision)
                if encoder_dict:
                    break

575
576
577
    if not encoder_dict:
        return None

578
579
    logger.info("Found sentence-transformers tokenize configuration.")

580
581
582
583
584
    if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
        return encoder_dict
    return None


585
def maybe_register_config_serialize_by_value() -> None:
586
587
    """Try to register HF model configuration class to serialize by value

588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
        If trust_remote_code is set, and the model's config file specifies an
        `AutoConfig` class, then the config class is typically an instance of
        a custom class imported from the HF modules cache.

        Examples:

        >>> from transformers import AutoConfig
        >>> klass = AutoConfig.from_pretrained('meta-llama/Meta-Llama-3-8B', trust_remote_code=True)
        >>> klass.__class__ # transformers.models.llama.configuration_llama.LlamaConfig
        >>> import transformers_modules # error, not initialized
        >>> klass = AutoConfig.from_pretrained('deepseek-ai/DeepSeek-V2.5', trust_remote_code=True)
        >>> import transformers_modules # success, initialized
        >>> klass.__class__ # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config

        In the DeepSeek example, the config class is an instance of a custom
        class that is not serializable by default. This class will not be
        importable in spawned workers, and won't exist at all on
        other nodes, which breaks serialization of the config.
606
607
608
609

        In this function we tell the cloudpickle serialization library to pass
        instances of these generated classes by value instead of by reference,
        i.e. the class definition is serialized along with its data so that the
610
        class module does not need to be importable on the receiving end.
611
612

        See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
613
    """ # noqa
614
615
616
    try:
        import transformers_modules
    except ImportError:
617
        # the config does not need trust_remote_code
618
619
620
621
622
623
624
625
626
627
628
629
        return

    try:
        import cloudpickle
        cloudpickle.register_pickle_by_value(transformers_modules)

        # ray vendors its own version of cloudpickle
        from vllm.executor.ray_utils import ray
        if ray:
            ray.cloudpickle.register_pickle_by_value(transformers_modules)

        # multiprocessing uses pickle to serialize arguments when using spawn
630
        # Here we get pickle to use cloudpickle to serialize config objects
631
632
633
634
635
636
        # that contain instances of the custom config class to avoid
        # serialization problems if the generated module (and model) has a `.`
        # in its name
        import multiprocessing
        import pickle

637
        from vllm.config import VllmConfig
638

639
640
        def _reduce_config(config: VllmConfig):
            return (pickle.loads, (cloudpickle.dumps(config), ))
641

642
        multiprocessing.reducer.register(VllmConfig, _reduce_config)
643
644
645
646
647
648
649
650
651
652

    except Exception as e:
        logger.warning(
            "Unable to register remote classes used by"
            " trust_remote_code with by-value serialization. This may"
            " lead to a later error. If remote code is not needed"
            " remove `--trust-remote-code`",
            exc_info=e)


653
def load_params_config(model: Union[str, Path], revision: Optional[str],
654
                       **kwargs) -> PretrainedConfig:
655
656
657
658
659
    # This function loads a params.json config which
    # should be used when loading models in mistral format

    config_file_name = "params.json"

660
    config_dict = get_hf_file_to_dict(config_file_name, model, revision)
661
662
663
664
665
    if config_dict is None:
        raise ValueError(
            f"Failed to load mistral '{config_file_name}' config for model "
            f"{model}. Please check if the model is a mistral-format model "
            f"and if the config file exists.")
666
    assert isinstance(config_dict, dict)
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682

    config_mapping = {
        "dim": "hidden_size",
        "norm_eps": "rms_norm_eps",
        "n_kv_heads": "num_key_value_heads",
        "n_layers": "num_hidden_layers",
        "n_heads": "num_attention_heads",
        "hidden_dim": "intermediate_size",
    }

    def recurse_elems(elem: Any):
        if isinstance(elem, dict):
            config_dict = {}
            for key, value in elem.items():
                key = config_mapping.get(key, key)
                config_dict[key] = recurse_elems(value)
683
684

            return config_dict
685
686
687
688
689
690
691
        else:
            return elem

    config_dict["model_type"] = config_dict.get("model_type", "transformer")
    config_dict["hidden_act"] = config_dict.get("activation", "silu")
    config_dict["tie_word_embeddings"] = config_dict.get(
        "tie_embeddings", False)
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709

    if config_dict.get("max_position_embeddings") is None:
        max_position_embeddings = 128_000
        try:
            trust_remote_code_val = kwargs.get("trust_remote_code", False)
            hf_config = get_config(model=model,
                                   trust_remote_code=trust_remote_code_val,
                                   revision=revision,
                                   config_format=ConfigFormat.HF)
            if hf_value := hf_config.get_text_config().max_position_embeddings:
                max_position_embeddings = hf_value
        except Exception as e:
            logger.warning(
                "The params.json file is missing 'max_position_embeddings'"
                " and could not get a value from the HF config."
                " Defaulting to 128000",
                exc_info=e)
        config_dict["max_position_embeddings"] = max_position_embeddings
710

711
712
713
714
715
716
717
718
    if config_dict.get("quantization") is not None:
        quantization = config_dict.get("quantization", {})
        if quantization.get("qformat_weight") == "fp8_e4m3":
            # This maps to the FP8 static per-tensor quantization scheme
            quantization_config = {
                "quant_method": "fp8",
                "activation_scheme": "static"
            }
719
720
721
        elif quantization.get("quant_method") == "compressed-tensors":
            # Pass through the quantization config to compressed-tensors
            quantization_config = quantization
722
723
724
725
726
727
728
729
730
731
        else:
            raise ValueError(
                f"Found unknown quantization='{quantization}' in config")

        config_dict["quantization_config"] = quantization_config

    config_type: Literal["text",
                         "multimodal"] = "multimodal" if config_dict.get(
                             "vision_encoder") is not None else "text"

Patrick von Platen's avatar
Patrick von Platen committed
732
733
734
735
736
    if config_dict.get("moe") is not None:
        config_dict["architectures"] = ["MixtralForCausalLM"]
    else:
        config_dict["architectures"] = ["MistralForCausalLM"]

737
    if config_type == "multimodal":
Patrick von Platen's avatar
Patrick von Platen committed
738
        multimodal_config = config_dict.pop("vision_encoder")
739
        quantization_config = config_dict.get("quantization_config", {})
740

Patrick von Platen's avatar
Patrick von Platen committed
741
742
743
744
745
746
        config_dict = {
            "text_config": config_dict,
            "vision_config": multimodal_config
        }
        config_dict["architectures"] = ["PixtralForConditionalGeneration"]
        config_dict["model_type"] = "pixtral"
747
748
        if quantization_config:
            config_dict["quantization_config"] = quantization_config
Patrick von Platen's avatar
Patrick von Platen committed
749

750
751
    config_dict.update(kwargs)

752
753
754
755
756
757
758
759
760
761
    config_dict = recurse_elems(config_dict)

    # transform to HF config format
    if config_type == "multimodal":
        config_dict["text_config"] = PretrainedConfig(
            **config_dict["text_config"])
        config_dict["vision_config"] = PretrainedConfig(
            **config_dict["vision_config"])

    return PretrainedConfig(**config_dict)
762
763


764
765
def get_hf_image_processor_config(
    model: Union[str, Path],
766
    hf_token: Optional[Union[bool, str]] = None,
767
768
    revision: Optional[str] = None,
    **kwargs,
769
) -> dict[str, Any]:
770
771
772
    # ModelScope does not provide an interface for image_processor
    if VLLM_USE_MODELSCOPE:
        return dict()
773
    # Separate model folder from file path for GGUF models
774
    if check_gguf_file(model):
775
        model = Path(model).parent
776
777
778
779
    return get_image_processor_config(model,
                                      token=hf_token,
                                      revision=revision,
                                      **kwargs)
780
781


782
783
def get_hf_text_config(config: PretrainedConfig):
    """Get the "sub" config relevant to llm for multi modal models.
784
    No op for pure text models.
785
    """
786
787
    # This block should be unnecessary after https://github.com/huggingface/transformers/pull/37517
    if hasattr(config, "thinker_config"):
788
789
790
791
        # TODO(suyang.fy): Refactor code.
        #  For Qwen2.5-Omni, change hf_text_config to
        #  thinker_config.text_config.
        return config.thinker_config.text_config
792
793
794
795
796
797
798
799
800
801

    text_config = config.get_text_config()

    if text_config is not config:
        # The code operates under the assumption that text_config should have
        # `num_attention_heads` (among others). Assert here to fail early
        # if transformers config doesn't align with this assumption.
        assert hasattr(text_config, "num_attention_heads")

    return text_config
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823


def try_get_generation_config(
    model: str,
    trust_remote_code: bool,
    revision: Optional[str] = None,
) -> Optional[GenerationConfig]:
    try:
        return GenerationConfig.from_pretrained(
            model,
            revision=revision,
        )
    except OSError:  # Not found
        try:
            config = get_config(
                model,
                trust_remote_code=trust_remote_code,
                revision=revision,
            )
            return GenerationConfig.from_model_config(config)
        except OSError:  # Not found
            return None
824
825
826
827
828
829
830
831
832
833
834
835
836


def get_cross_encoder_activation_function(config: PretrainedConfig):
    if (hasattr(config, "sbert_ce_default_activation_function")
            and config.sbert_ce_default_activation_function is not None):

        function_name = config.sbert_ce_default_activation_function
        assert function_name.startswith("torch.nn.modules."), \
            "Loading of activation functions is restricted to " \
            "torch.nn.modules for security reasons"
        return resolve_obj_by_qualname(function_name)()
    else:
        return nn.Sigmoid() if config.num_labels == 1 else nn.Identity()