registry.py 22.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
3
4
5
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
6
import importlib
7
import os
8
import pickle
9
10
import subprocess
import sys
11
import tempfile
12
13
14
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from functools import lru_cache
15
16
from typing import (AbstractSet, Callable, Dict, List, Optional, Tuple, Type,
                    TypeVar, Union)
17

18
import cloudpickle
19
20
21
import torch.nn as nn

from vllm.logger import init_logger
22
from vllm.utils import is_in_doc_build
23

24
from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
25
                         supports_cross_encoding, supports_multimodal,
26
                         supports_pp, supports_transcription, supports_v0_only)
27
from .interfaces_base import is_text_generation_model
28
29
30

logger = init_logger(__name__)

31
# yapf: disable
32
33
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
34
35
36
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
37
38
39
40
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
41
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
42
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
43
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
44
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
45
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
46
47
48
49
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
    "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
50
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
51
52
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
53
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
54
55
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
56
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
57
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
58
59
60
61
62
63
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
64
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
65
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
66
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
67
68
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
69
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
70
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
71
72
73
74
75
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
76
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
77
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
78
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
79
80
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
81
82
83
84
85
86
87
88
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
89
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
90
91
92
93
94
95
96
97
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
98
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
99
100
101
102
103
104
105
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
106
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
107
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
108
109
110
    # [Encoder-decoder]
    "BartModel": ("bart", "BartForConditionalGeneration"),
    "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
111
112
113
}

_EMBEDDING_MODELS = {
114
    # [Text-only]
115
    "BertModel": ("bert", "BertEmbeddingModel"),
116
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
117
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
118
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
119
    "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
120
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
121
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
122
    "GritLM": ("gritlm", "GritLM"),
123
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
124
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
125
    "LlamaModel": ("llama", "LlamaForCausalLM"),
126
127
128
129
130
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
131
    "MistralModel": ("llama", "LlamaForCausalLM"),
132
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
133
134
    "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
135
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
136
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
137
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
138
    # [Multimodal]
Cyrus Leung's avatar
Cyrus Leung committed
139
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
140
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
141
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
142
143
    # [Auto-converted (see adapters.py)]
    "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
144
145
146
147
    # Technically PrithviGeoSpatialMAE is a model that works on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
    # models for the time being.
    "PrithviGeoSpatialMAE": ("prithvi_geospatial_mae", "PrithviGeoSpatialMAE"),
148
149
}

150
151
152
153
154
155
156
157
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
}

158
_MULTIMODAL_MODELS = {
159
    # [Decoder-only]
160
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
161
162
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
163
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
164
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
165
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
166
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
167
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
168
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
169
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
170
171
172
173
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
174
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
175
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
176
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
177
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
178
179
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
180
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
181
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
182
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
183
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
184
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
185
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
186
    "UltravoxModel": ("ultravox", "UltravoxModel"),
187
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
188
    # [Encoder-decoder]
189
    "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"),  # noqa: E501
190
    "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"),  # noqa: E501
191
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
192
}
193
194
195

_SPECULATIVE_DECODING_MODELS = {
    "EAGLEModel": ("eagle", "EAGLE"),
196
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
197
198
    "MedusaModel": ("medusa", "Medusa"),
    "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
199
}
200
201
202
203

_FALLBACK_MODEL = {
    "TransformersModel": ("transformers", "TransformersModel"),
}
204
# yapf: enable
205

206
_VLLM_MODELS = {
207
    **_TEXT_GENERATION_MODELS,
208
    **_EMBEDDING_MODELS,
209
    **_CROSS_ENCODER_MODELS,
210
    **_MULTIMODAL_MODELS,
211
    **_SPECULATIVE_DECODING_MODELS,
212
    **_FALLBACK_MODEL,
213
214
}

215
216
217
218
219
220
221
222
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
_SUBPROCESS_COMMAND = [
    sys.executable, "-m", "vllm.model_executor.models.registry"
]

223

224
225
@dataclass(frozen=True)
class _ModelInfo:
226
    architecture: str
227
    is_text_generation_model: bool
228
    is_pooling_model: bool
229
    supports_cross_encoding: bool
230
231
    supports_multimodal: bool
    supports_pp: bool
232
233
    has_inner_state: bool
    is_attention_free: bool
234
    is_hybrid: bool
235
    supports_transcription: bool
236
    supports_v0_only: bool
237
238

    @staticmethod
239
240
    def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
        return _ModelInfo(
241
            architecture=model.__name__,
242
            is_text_generation_model=is_text_generation_model(model),
243
            is_pooling_model=True,  # Can convert any model into a pooling model
244
            supports_cross_encoding=supports_cross_encoding(model),
245
246
            supports_multimodal=supports_multimodal(model),
            supports_pp=supports_pp(model),
247
248
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
249
            is_hybrid=is_hybrid(model),
250
251
252
            supports_transcription=supports_transcription(model),
            supports_v0_only=supports_v0_only(model),
        )
253
254


255
class _BaseRegisteredModel(ABC):
256

257
258
259
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
260

261
262
263
    @abstractmethod
    def load_model_cls(self) -> Type[nn.Module]:
        raise NotImplementedError
264
265


266
267
268
269
270
271
272
273
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
    model_cls: Type[nn.Module]
274
275

    @staticmethod
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
    def from_model_cls(model_cls: Type[nn.Module]):
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

    def load_model_cls(self) -> Type[nn.Module]:
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
    module_name: str
    class_name: str

    # Performed in another process to avoid initializing CUDA
    def inspect_model_cls(self) -> _ModelInfo:
        return _run_in_subprocess(
            lambda: _ModelInfo.from_model_cls(self.load_model_cls()))

    def load_model_cls(self) -> Type[nn.Module]:
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[Type[nn.Module]]:
312
    from vllm.platforms import current_platform
313
    current_platform.verify_model_arch(model_arch)
314
315
316
317
318
319
    try:
        return model.load_model_cls()
    except Exception:
        logger.exception("Error in loading model architecture '%s'",
                         model_arch)
        return None
320
321


322
323
324
325
326
327
328
329
330
331
332
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
        logger.exception("Error in inspecting model architecture '%s'",
                         model_arch)
        return None
333
334


335
336
337
338
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
    models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
339

340
341
    def get_supported_archs(self) -> AbstractSet[str]:
        return self.models.keys()
342

343
344
345
346
347
    def register_model(
        self,
        model_arch: str,
        model_cls: Union[Type[nn.Module], str],
    ) -> None:
348
349
350
351
352
353
354
355
356
357
358
        """
        Register an external model to be used in vLLM.

        :code:`model_cls` can be either:

        - A :class:`torch.nn.Module` class directly referencing the model.
        - A string in the format :code:`<module>:<class>` which can be used to
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
          :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
        """
359
360
361
362
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

363
        if model_arch in self.models:
364
365
366
            logger.warning(
                "Model architecture %s is already registered, and will be "
                "overwritten by the new model class %s.", model_arch,
367
368
369
370
371
372
373
                model_cls)

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
374

375
            model = _LazyRegisteredModel(*split_str)
376
377
        elif isinstance(model_cls, type) and (is_in_doc_build() or issubclass(
                model_cls, nn.Module)):
378
            model = _RegisteredModel.from_model_cls(model_cls)
379
380
381
382
        else:
            msg = ("`model_cls` should be a string or PyTorch model class, "
                   f"not a {type(model_arch)}")
            raise TypeError(msg)
383

384
        self.models[model_arch] = model
385

386
387
    def _raise_for_unsupported(self, architectures: List[str]):
        all_supported_archs = self.get_supported_archs()
388

389
390
391
392
393
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
                "to be inspected. Please check the logs for more details.")

394
395
396
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
            f"Supported architectures: {all_supported_archs}")
397

398
399
400
401
    def _try_load_model_cls(self,
                            model_arch: str) -> Optional[Type[nn.Module]]:
        if model_arch not in self.models:
            return None
402

403
        return _try_load_model_cls(model_arch, self.models[model_arch])
404

405
406
407
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
        if model_arch not in self.models:
            return None
408

409
        return _try_inspect_model_cls(model_arch, self.models[model_arch])
410

411
412
413
414
    def _normalize_archs(
        self,
        architectures: Union[str, List[str]],
    ) -> List[str]:
415
416
417
418
419
        if isinstance(architectures, str):
            architectures = [architectures]
        if not architectures:
            logger.warning("No model architectures are specified")

420
421
422
423
424
425
        normalized_arch = []
        for model in architectures:
            if model not in self.models:
                model = "TransformersModel"
            normalized_arch.append(model)
        return normalized_arch
426

427
428
429
    def inspect_model_cls(
        self,
        architectures: Union[str, List[str]],
430
    ) -> Tuple[_ModelInfo, str]:
431
        architectures = self._normalize_archs(architectures)
432

433
434
435
        for arch in architectures:
            model_info = self._try_inspect_model_cls(arch)
            if model_info is not None:
436
                return (model_info, arch)
437

438
        return self._raise_for_unsupported(architectures)
439

440
441
442
443
444
    def resolve_model_cls(
        self,
        architectures: Union[str, List[str]],
    ) -> Tuple[Type[nn.Module], str]:
        architectures = self._normalize_archs(architectures)
445

446
447
448
449
        for arch in architectures:
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
450

451
        return self._raise_for_unsupported(architectures)
452

453
454
455
456
    def is_text_generation_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
457
458
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_text_generation_model
459

460
    def is_pooling_model(
461
462
463
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
464
        model_cls, _ = self.inspect_model_cls(architectures)
465
        return model_cls.is_pooling_model
466

467
468
469
470
    def is_cross_encoder_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
471
472
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_cross_encoding
473

474
475
476
477
    def is_multimodal_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
478
479
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_multimodal
480
481
482
483
484

    def is_pp_supported_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
485
486
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_pp
487

488
489
490
491
492
493
    def model_has_inner_state(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.has_inner_state
494

495
496
497
498
499
500
    def is_attention_free_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_attention_free
501

502
503
504
505
506
507
508
    def is_hybrid_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_hybrid

509
510
511
512
513
514
515
    def is_transcription_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_transcription

516
517
518
519
520
521
522
    def is_v1_compatible(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return not model_cls.supports_v0_only

523
524

ModelRegistry = _ModelRegistry({
525
526
    model_arch:
    _LazyRegisteredModel(
527
528
529
530
531
532
533
534
535
536
        module_name=f"vllm.model_executor.models.{mod_relname}",
        class_name=cls_name,
    )
    for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
})

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
537
538
539
540
541
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

542
        # `cloudpickle` allows pickling lambda functions directly
543
        input_bytes = cloudpickle.dumps((fn, output_filepath))
544
545
546

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
547
548
549
        returned = subprocess.run(_SUBPROCESS_COMMAND,
                                  input=input_bytes,
                                  capture_output=True)
550
551
552
553
554
555
556
557
558

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
            raise RuntimeError(f"Error raised in subprocess:\n"
                               f"{returned.stderr.decode()}") from e

559
        with open(output_filepath, "rb") as f:
560
561
562
563
564
565
566
567
568
569
570
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
571
572
573

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
574
575
576


if __name__ == "__main__":
577
    _run()