registry.py 22.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
3
4
5
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
6
import importlib
7
import os
8
import pickle
9
10
import subprocess
import sys
11
import tempfile
12
13
14
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from functools import lru_cache
15
16
from typing import (AbstractSet, Callable, Dict, List, Optional, Tuple, Type,
                    TypeVar, Union)
17

18
import cloudpickle
19
20
21
import torch.nn as nn

from vllm.logger import init_logger
22
from vllm.utils import is_in_doc_build
23

24
from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
25
                         supports_cross_encoding, supports_multimodal,
26
                         supports_pp, supports_transcription, supports_v0_only)
27
from .interfaces_base import is_text_generation_model
28
29
30

logger = init_logger(__name__)

31
# yapf: disable
32
33
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
34
35
36
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
37
38
39
40
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
41
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
42
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
43
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
44
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
45
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
46
47
48
49
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
    "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
50
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
51
52
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
53
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
54
55
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
56
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
57
58
59
60
61
62
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
63
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
64
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
65
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
66
67
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
68
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
69
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
70
71
72
73
74
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
75
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
76
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
77
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
78
79
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
80
81
82
83
84
85
86
87
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
88
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
89
90
91
92
93
94
95
96
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
97
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
98
99
100
101
102
103
104
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
105
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
106
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
107
108
109
    # [Encoder-decoder]
    "BartModel": ("bart", "BartForConditionalGeneration"),
    "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
110
111
112
}

_EMBEDDING_MODELS = {
113
    # [Text-only]
114
    "BertModel": ("bert", "BertEmbeddingModel"),
115
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
116
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
117
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
118
    "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
119
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
120
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
121
    "GritLM": ("gritlm", "GritLM"),
122
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
123
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
124
    "LlamaModel": ("llama", "LlamaForCausalLM"),
125
126
127
128
129
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
130
    "MistralModel": ("llama", "LlamaForCausalLM"),
131
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
132
133
    "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
134
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
135
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
136
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
137
    # [Multimodal]
Cyrus Leung's avatar
Cyrus Leung committed
138
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
139
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
140
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
141
142
    # [Auto-converted (see adapters.py)]
    "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
143
144
145
146
    # Technically PrithviGeoSpatialMAE is a model that works on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
    # models for the time being.
    "PrithviGeoSpatialMAE": ("prithvi_geospatial_mae", "PrithviGeoSpatialMAE"),
147
148
}

149
150
151
152
153
154
155
156
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
}

157
_MULTIMODAL_MODELS = {
158
    # [Decoder-only]
159
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
160
161
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
162
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
163
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
164
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
165
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
166
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
167
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
168
169
170
171
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
172
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
173
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
174
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
175
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
176
177
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
178
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
179
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
180
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
181
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
182
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
183
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
184
    "UltravoxModel": ("ultravox", "UltravoxModel"),
185
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
186
    # [Encoder-decoder]
187
    "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"),  # noqa: E501
188
    "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"),  # noqa: E501
189
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
190
}
191
192
193

_SPECULATIVE_DECODING_MODELS = {
    "EAGLEModel": ("eagle", "EAGLE"),
194
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
195
196
    "MedusaModel": ("medusa", "Medusa"),
    "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
197
}
198
199
200
201

_FALLBACK_MODEL = {
    "TransformersModel": ("transformers", "TransformersModel"),
}
202
# yapf: enable
203

204
_VLLM_MODELS = {
205
    **_TEXT_GENERATION_MODELS,
206
    **_EMBEDDING_MODELS,
207
    **_CROSS_ENCODER_MODELS,
208
    **_MULTIMODAL_MODELS,
209
    **_SPECULATIVE_DECODING_MODELS,
210
    **_FALLBACK_MODEL,
211
212
}

213
214
215
216
217
218
219
220
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
_SUBPROCESS_COMMAND = [
    sys.executable, "-m", "vllm.model_executor.models.registry"
]

221

222
223
@dataclass(frozen=True)
class _ModelInfo:
224
    architecture: str
225
    is_text_generation_model: bool
226
    is_pooling_model: bool
227
    supports_cross_encoding: bool
228
229
    supports_multimodal: bool
    supports_pp: bool
230
231
    has_inner_state: bool
    is_attention_free: bool
232
    is_hybrid: bool
233
    supports_transcription: bool
234
    supports_v0_only: bool
235
236

    @staticmethod
237
238
    def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
        return _ModelInfo(
239
            architecture=model.__name__,
240
            is_text_generation_model=is_text_generation_model(model),
241
            is_pooling_model=True,  # Can convert any model into a pooling model
242
            supports_cross_encoding=supports_cross_encoding(model),
243
244
            supports_multimodal=supports_multimodal(model),
            supports_pp=supports_pp(model),
245
246
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
247
            is_hybrid=is_hybrid(model),
248
249
250
            supports_transcription=supports_transcription(model),
            supports_v0_only=supports_v0_only(model),
        )
251
252


253
class _BaseRegisteredModel(ABC):
254

255
256
257
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
258

259
260
261
    @abstractmethod
    def load_model_cls(self) -> Type[nn.Module]:
        raise NotImplementedError
262
263


264
265
266
267
268
269
270
271
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
    model_cls: Type[nn.Module]
272
273

    @staticmethod
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
    def from_model_cls(model_cls: Type[nn.Module]):
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

    def load_model_cls(self) -> Type[nn.Module]:
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
    module_name: str
    class_name: str

    # Performed in another process to avoid initializing CUDA
    def inspect_model_cls(self) -> _ModelInfo:
        return _run_in_subprocess(
            lambda: _ModelInfo.from_model_cls(self.load_model_cls()))

    def load_model_cls(self) -> Type[nn.Module]:
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[Type[nn.Module]]:
310
    from vllm.platforms import current_platform
311
    current_platform.verify_model_arch(model_arch)
312
313
314
315
316
317
    try:
        return model.load_model_cls()
    except Exception:
        logger.exception("Error in loading model architecture '%s'",
                         model_arch)
        return None
318
319


320
321
322
323
324
325
326
327
328
329
330
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
        logger.exception("Error in inspecting model architecture '%s'",
                         model_arch)
        return None
331
332


333
334
335
336
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
    models: Dict[str, _BaseRegisteredModel] = field(default_factory=dict)
337

338
339
    def get_supported_archs(self) -> AbstractSet[str]:
        return self.models.keys()
340

341
342
343
344
345
    def register_model(
        self,
        model_arch: str,
        model_cls: Union[Type[nn.Module], str],
    ) -> None:
346
347
348
349
350
351
352
353
354
355
356
        """
        Register an external model to be used in vLLM.

        :code:`model_cls` can be either:

        - A :class:`torch.nn.Module` class directly referencing the model.
        - A string in the format :code:`<module>:<class>` which can be used to
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
          :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
        """
357
358
359
360
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

361
        if model_arch in self.models:
362
363
364
            logger.warning(
                "Model architecture %s is already registered, and will be "
                "overwritten by the new model class %s.", model_arch,
365
366
367
368
369
370
371
                model_cls)

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
372

373
            model = _LazyRegisteredModel(*split_str)
374
375
        elif isinstance(model_cls, type) and (is_in_doc_build() or issubclass(
                model_cls, nn.Module)):
376
            model = _RegisteredModel.from_model_cls(model_cls)
377
378
379
380
        else:
            msg = ("`model_cls` should be a string or PyTorch model class, "
                   f"not a {type(model_arch)}")
            raise TypeError(msg)
381

382
        self.models[model_arch] = model
383

384
385
    def _raise_for_unsupported(self, architectures: List[str]):
        all_supported_archs = self.get_supported_archs()
386

387
388
389
390
391
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
                "to be inspected. Please check the logs for more details.")

392
393
394
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
            f"Supported architectures: {all_supported_archs}")
395

396
397
398
399
    def _try_load_model_cls(self,
                            model_arch: str) -> Optional[Type[nn.Module]]:
        if model_arch not in self.models:
            return None
400

401
        return _try_load_model_cls(model_arch, self.models[model_arch])
402

403
404
405
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
        if model_arch not in self.models:
            return None
406

407
        return _try_inspect_model_cls(model_arch, self.models[model_arch])
408

409
410
411
412
    def _normalize_archs(
        self,
        architectures: Union[str, List[str]],
    ) -> List[str]:
413
414
415
416
417
        if isinstance(architectures, str):
            architectures = [architectures]
        if not architectures:
            logger.warning("No model architectures are specified")

418
419
420
421
422
423
        normalized_arch = []
        for model in architectures:
            if model not in self.models:
                model = "TransformersModel"
            normalized_arch.append(model)
        return normalized_arch
424

425
426
427
    def inspect_model_cls(
        self,
        architectures: Union[str, List[str]],
428
    ) -> Tuple[_ModelInfo, str]:
429
        architectures = self._normalize_archs(architectures)
430

431
432
433
        for arch in architectures:
            model_info = self._try_inspect_model_cls(arch)
            if model_info is not None:
434
                return (model_info, arch)
435

436
        return self._raise_for_unsupported(architectures)
437

438
439
440
441
442
    def resolve_model_cls(
        self,
        architectures: Union[str, List[str]],
    ) -> Tuple[Type[nn.Module], str]:
        architectures = self._normalize_archs(architectures)
443

444
445
446
447
        for arch in architectures:
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
448

449
        return self._raise_for_unsupported(architectures)
450

451
452
453
454
    def is_text_generation_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
455
456
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_text_generation_model
457

458
    def is_pooling_model(
459
460
461
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
462
        model_cls, _ = self.inspect_model_cls(architectures)
463
        return model_cls.is_pooling_model
464

465
466
467
468
    def is_cross_encoder_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
469
470
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_cross_encoding
471

472
473
474
475
    def is_multimodal_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
476
477
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_multimodal
478
479
480
481
482

    def is_pp_supported_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
483
484
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_pp
485

486
487
488
489
490
491
    def model_has_inner_state(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.has_inner_state
492

493
494
495
496
497
498
    def is_attention_free_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_attention_free
499

500
501
502
503
504
505
506
    def is_hybrid_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.is_hybrid

507
508
509
510
511
512
513
    def is_transcription_model(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return model_cls.supports_transcription

514
515
516
517
518
519
520
    def is_v1_compatible(
        self,
        architectures: Union[str, List[str]],
    ) -> bool:
        model_cls, _ = self.inspect_model_cls(architectures)
        return not model_cls.supports_v0_only

521
522

ModelRegistry = _ModelRegistry({
523
524
    model_arch:
    _LazyRegisteredModel(
525
526
527
528
529
530
531
532
533
534
        module_name=f"vllm.model_executor.models.{mod_relname}",
        class_name=cls_name,
    )
    for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
})

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
535
536
537
538
539
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

540
        # `cloudpickle` allows pickling lambda functions directly
541
        input_bytes = cloudpickle.dumps((fn, output_filepath))
542
543
544

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
545
546
547
        returned = subprocess.run(_SUBPROCESS_COMMAND,
                                  input=input_bytes,
                                  capture_output=True)
548
549
550
551
552
553
554
555
556

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
            raise RuntimeError(f"Error raised in subprocess:\n"
                               f"{returned.stderr.decode()}") from e

557
        with open(output_filepath, "rb") as f:
558
559
560
561
562
563
564
565
566
567
568
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
569
570
571

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
572
573
574


if __name__ == "__main__":
575
    _run()