registry.py 22.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
3
4
5
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
6
import importlib
7
import os
8
import pickle
9
10
import subprocess
import sys
11
import tempfile
12
13
14
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from functools import lru_cache
15
16
from typing import (AbstractSet, Callable, Dict, List, Optional, Tuple, Type,
                    TypeVar, Union)
17

18
import cloudpickle
19
20
21
import torch.nn as nn

from vllm.logger import init_logger
22
from vllm.utils import is_in_doc_build
23

24
from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
25
                         supports_cross_encoding, supports_multimodal,
26
                         supports_pp, supports_transcription, supports_v0_only)
27
from .interfaces_base import is_text_generation_model
28
29
30

logger = init_logger(__name__)

31
# yapf: disable
32
33
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
34
35
36
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
37
38
39
40
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
41
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
42
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
43
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
44
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
45
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
46
47
48
49
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
    "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
50
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
51
52
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
53
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
54
55
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
56
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
57
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
58
59
60
61
62
63
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
64
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
65
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
66
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
67
68
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
69
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
70
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
71
72
73
74
75
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
76
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
77
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
78
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
79
80
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
81
82
83
84
85
86
87
88
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
89
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
90
91
92
93
94
95
96
97
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
98
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
99
100
101
102
103
104
105
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
106
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
107
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
108
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
109
110
111
    # [Encoder-decoder]
    "BartModel": ("bart", "BartForConditionalGeneration"),
    "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
112
113
114
}

_EMBEDDING_MODELS = {
115
    # [Text-only]
116
    "BertModel": ("bert", "BertEmbeddingModel"),
117
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
118
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
119
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
120
    "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
121
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
122
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
123
    "GritLM": ("gritlm", "GritLM"),
124
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
125
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
126
    "LlamaModel": ("llama", "LlamaForCausalLM"),
127
128
129
130
131
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
132
    "MistralModel": ("llama", "LlamaForCausalLM"),
133
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
134
135
    "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
136
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
137
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
138
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
139
    # [Multimodal]
Cyrus Leung's avatar
Cyrus Leung committed
140
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
141
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
142
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
143
144
    # [Auto-converted (see adapters.py)]
    "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
145
146
147
148
    # Technically PrithviGeoSpatialMAE is a model that works on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
    # models for the time being.
    "PrithviGeoSpatialMAE": ("prithvi_geospatial_mae", "PrithviGeoSpatialMAE"),
149
150
}

151
152
153
154
155
156
157
158
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
}

159
_MULTIMODAL_MODELS = {
160
    # [Decoder-only]
161
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
162
163
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
164
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
165
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
166
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
167
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
168
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
169
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
170
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
171
172
173
174
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
175
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
176
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
177
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
178
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
179
180
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
181
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
182
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
183
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
184
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
185
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
186
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
187
    "UltravoxModel": ("ultravox", "UltravoxModel"),
188
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
189
    # [Encoder-decoder]
190
    "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"),  # noqa: E501
191
    "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"),  # noqa: E501
192
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
193
}
194
195
196

_SPECULATIVE_DECODING_MODELS = {
    "EAGLEModel": ("eagle", "EAGLE"),
197
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
198
199
    "MedusaModel": ("medusa", "Medusa"),
    "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
200
}
201
202
203
204

_FALLBACK_MODEL = {
    "TransformersModel": ("transformers", "TransformersModel"),
}
205
# yapf: enable
206

207
_VLLM_MODELS = {
208
    **_TEXT_GENERATION_MODELS,
209
    **_EMBEDDING_MODELS,
210
    **_CROSS_ENCODER_MODELS,
211
    **_MULTIMODAL_MODELS,
212
    **_SPECULATIVE_DECODING_MODELS,
213
    **_FALLBACK_MODEL,
214
215
}

216
217
218
219
220
221
222
223
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
_SUBPROCESS_COMMAND = [
    sys.executable, "-m", "vllm.model_executor.models.registry"
]

224

225
226
@dataclass(frozen=True)
class _ModelInfo:
227
    architecture: str
228
    is_text_generation_model: bool
229
    is_pooling_model: bool
230
    supports_cross_encoding: bool
231
232
    supports_multimodal: bool
    supports_pp: bool
233
234
    has_inner_state: bool
    is_attention_free: bool
235
    is_hybrid: bool
236
    supports_transcription: bool
237
    supports_v0_only: bool
238
239

    @staticmethod
240
241
    def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
        return _ModelInfo(
242
            architecture=model.__name__,
243
            is_text_generation_model=is_text_generation_model(model),
244
            is_pooling_model=True,  # Can convert any model into a pooling model
245
            supports_cross_encoding=supports_cross_encoding(model),
246
247
            supports_multimodal=supports_multimodal(model),
            supports_pp=supports_pp(model),
248
249
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
250
            is_hybrid=is_hybrid(model),
251
252
253
            supports_transcription=supports_transcription(model),
            supports_v0_only=supports_v0_only(model),
        )
254
255


256
class _BaseRegisteredModel(ABC):
257

258
259
260
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
261

262
263
264
    @abstractmethod
    def load_model_cls(self) -> Type[nn.Module]:
        raise NotImplementedError
265
266


267
268
269
270
271
272
273
274
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
    model_cls: Type[nn.Module]
275
276

    @staticmethod
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
    def from_model_cls(model_cls: Type[nn.Module]):
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

    def load_model_cls(self) -> Type[nn.Module]:
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
    module_name: str
    class_name: str

    # Performed in another process to avoid initializing CUDA
    def inspect_model_cls(self) -> _ModelInfo:
        return _run_in_subprocess(
            lambda: _ModelInfo.from_model_cls(self.load_model_cls()))

    def load_model_cls(self) -> Type[nn.Module]:
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[Type[nn.Module]]:
313
    from vllm.platforms import current_platform
314
    current_platform.verify_model_arch(model_arch)
315
316
317
318
319
320
    try:
        return model.load_model_cls()
    except Exception:
        logger.exception("Error in loading model architecture '%s'",
                         model_arch)
        return None
321
322


323
324
325
326
327
328
329
330
331
332
333
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
        logger.exception("Error in inspecting model architecture '%s'",
                         model_arch)
        return None
334
335


336
337
338
339
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
    models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
340

341
342
    def get_supported_archs(self) -> AbstractSet[str]:
        return self.models.keys()
343

344
345
346
347
348
    def register_model(
        self,
        model_arch: str,
        model_cls: Union[Type[nn.Module], str],
    ) -> None:
349
350
351
352
353
354
355
356
357
358
359
        """
        Register an external model to be used in vLLM.

        :code:`model_cls` can be either:

        - A :class:`torch.nn.Module` class directly referencing the model.
        - A string in the format :code:`<module>:<class>` which can be used to
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
          :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
        """
360
361
362
363
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

364
        if model_arch in self.models:
365
366
367
            logger.warning(
                "Model architecture %s is already registered, and will be "
                "overwritten by the new model class %s.", model_arch,
368
369
370
371
372
373
374
                model_cls)

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
375

376
            model = _LazyRegisteredModel(*split_str)
377
378
        elif isinstance(model_cls, type) and (is_in_doc_build() or issubclass(
                model_cls, nn.Module)):
379
            model = _RegisteredModel.from_model_cls(model_cls)
380
381
382
383
        else:
            msg = ("`model_cls` should be a string or PyTorch model class, "
                   f"not a {type(model_arch)}")
            raise TypeError(msg)
384

385
        self.models[model_arch] = model
386

387
388
    def _raise_for_unsupported(self, architectures: List[str]):
        all_supported_archs = self.get_supported_archs()
389

390
391
392
393
394
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
                "to be inspected. Please check the logs for more details.")

395
396
397
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
            f"Supported architectures: {all_supported_archs}")
398

399
400
401
402
    def _try_load_model_cls(self,
                            model_arch: str) -> Optional[Type[nn.Module]]:
        if model_arch not in self.models:
            return None
403

404
        return _try_load_model_cls(model_arch, self.models[model_arch])
405

406
407
408
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
        if model_arch not in self.models:
            return None
409

410
        return _try_inspect_model_cls(model_arch, self.models[model_arch])
411

412
413
414
415
    def _normalize_archs(
        self,
        architectures: Union[str, List[str]],
    ) -> List[str]:
416
417
418
419
420
        if isinstance(architectures, str):
            architectures = [architectures]
        if not architectures:
            logger.warning("No model architectures are specified")

421
422
423
424
425
426
427
        # filter out support architectures
        normalized_arch = list(
            filter(lambda model: model in self.models, architectures))

        # make sure Transformers fallback are put at the last
        if len(normalized_arch) != len(architectures):
            normalized_arch.append("TransformersModel")
428
        return normalized_arch
429

430
431
432
    def inspect_model_cls(
        self,
        architectures: Union[str, List[str]],
433
    ) -> Tuple[_ModelInfo, str]:
434
        architectures = self._normalize_archs(architectures)
435

436
437
438
        for arch in architectures:
            model_info = self._try_inspect_model_cls(arch)
            if model_info is not None:
439
                return (model_info, arch)
440

441
        return self._raise_for_unsupported(architectures)
442

443
444
445
446
447
    def resolve_model_cls(
        self,
        architectures: Union[str, List[str]],
    ) -> Tuple[Type[nn.Module], str]:
        architectures = self._normalize_archs(architectures)
448

449
450
451
452
        for arch in architectures:
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
453

454
        return self._raise_for_unsupported(architectures)
455

456
457
458
459
    def is_text_generation_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
460
461
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_text_generation_model
462

463
    def is_pooling_model(
464
465
466
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
467
        model_cls, _ = self.inspect_model_cls(architectures)
468
        return model_cls.is_pooling_model
469

470
471
472
473
    def is_cross_encoder_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
474
475
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_cross_encoding
476

477
478
479
480
    def is_multimodal_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
481
482
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_multimodal
483
484
485
486
487

    def is_pp_supported_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
488
489
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_pp
490

491
492
493
494
495
496
    def model_has_inner_state(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.has_inner_state
497

498
499
500
501
502
503
    def is_attention_free_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_attention_free
504

505
506
507
508
509
510
511
    def is_hybrid_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_hybrid

512
513
514
515
516
517
518
    def is_transcription_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_transcription

519
520
521
522
523
524
525
    def is_v1_compatible(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return not model_cls.supports_v0_only

526
527

ModelRegistry = _ModelRegistry({
528
529
    model_arch:
    _LazyRegisteredModel(
530
531
532
533
534
535
536
537
538
539
        module_name=f"vllm.model_executor.models.{mod_relname}",
        class_name=cls_name,
    )
    for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
})

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
540
541
542
543
544
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

545
        # `cloudpickle` allows pickling lambda functions directly
546
        input_bytes = cloudpickle.dumps((fn, output_filepath))
547
548
549

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
550
551
552
        returned = subprocess.run(_SUBPROCESS_COMMAND,
                                  input=input_bytes,
                                  capture_output=True)
553
554
555
556
557
558
559
560
561

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
            raise RuntimeError(f"Error raised in subprocess:\n"
                               f"{returned.stderr.decode()}") from e

562
        with open(output_filepath, "rb") as f:
563
564
565
566
567
568
569
570
571
572
573
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
574
575
576

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
577
578
579


if __name__ == "__main__":
580
    _run()