registry.py 42.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import hashlib
9
import importlib
10
import json
11
import os
12
import pickle
13
14
import subprocess
import sys
15
import tempfile
16
from abc import ABC, abstractmethod
17
from collections.abc import Set
18
from dataclasses import asdict, dataclass, field
19
from functools import lru_cache
20
from pathlib import Path
21
from typing import Callable, Optional, TypeVar, Union
22
23

import torch.nn as nn
24
import transformers
25

26
from vllm import envs
27
28
29
30
31
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
32
from vllm.logger import init_logger
33
from vllm.logging_utils import logtime
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module

from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
    supports_cross_encoding,
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
    supports_v0_only,
)
from .interfaces_base import (
    get_default_pooling_type,
    is_pooling_model,
    is_text_generation_model,
)
54
55
56

logger = init_logger(__name__)

57
# yapf: disable
58
59
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
60
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
61
62
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
63
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
64
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
65
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
66
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
67
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
68
69
70
71
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
72
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
73
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
74
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
75
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
76
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
77
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
78
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
79
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
80
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
81
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
82
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
83
84
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
85
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
86
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
87
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
88
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
89
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
90
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
91
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
92
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
93
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
94
95
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
96
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
97
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
98
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
99
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
100
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
101
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
102
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
103
104
105
106
107
108
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
109
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),   # noqa: E501
110
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
111
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
112
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
113
114
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
115
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
116
117
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
118
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
119
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
120
121
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
122
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
123
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
124
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),  # noqa: E501
125
126
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
127
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
128
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
129
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
Dhia Eddine Rhaiem's avatar
Dhia Eddine Rhaiem committed
130
    "FalconH1ForCausalLM":("falcon_h1", "FalconH1ForCausalLM"),
131
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
132
133
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
134
135
136
137
138
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
139
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
140
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
141
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
142
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
143
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
144
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
145
146
147
148
149
150
151
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
152
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
153
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
154
155
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
156
157
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
158
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
159
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Song's avatar
Song committed
160
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
161
162
163
164
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
165
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
166
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
167
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
168
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
169
170
171
}

_EMBEDDING_MODELS = {
172
    # [Text-only]
173
    "BertModel": ("bert", "BertEmbeddingModel"),
174
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
175
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
176
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
177
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
178
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
179
    "GritLM": ("gritlm", "GritLM"),
180
181
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
182
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
183
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
184
    "LlamaModel": ("llama", "LlamaForCausalLM"),
185
186
187
188
189
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
190
    "MistralModel": ("llama", "LlamaForCausalLM"),
191
    "ModernBertModel": ("modernbert", "ModernBertModel"),
192
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
193
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
194
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
195
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
196
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
197
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
198
199
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
200
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
201
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
202
    # [Multimodal]
203
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
Cyrus Leung's avatar
Cyrus Leung committed
204
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
205
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
206
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
207
208
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
209
    # models for the time being.
210
211
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
212
213
}

214
215
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
216
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
217
218
219
220
    "GteNewForSequenceClassification": ("bert_with_rope",
                                        "GteNewForSequenceClassification"),
    "ModernBertForSequenceClassification": ("modernbert",
                                            "ModernBertForSequenceClassification"),
221
222
223
224
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
225
    # [Auto-converted (see adapters.py)]
226
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"), # noqa: E501,
227
228
}

229
_MULTIMODAL_MODELS = {
230
    # [Decoder-only]
231
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
Jennifer Zhao's avatar
Jennifer Zhao committed
232
    "AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"),  # noqa: E501
233
234
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
235
    "Cohere2VisionForConditionalGeneration": ("cohere2_vision", "Cohere2VisionForConditionalGeneration"),  # noqa: E501
236
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
237
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
238
    "Ernie4_5_VLMoeForConditionalGeneration": ("ernie45_vl", "Ernie4_5_VLMoeForConditionalGeneration"),  # noqa: E501
239
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
240
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
241
    "Gemma3nForConditionalGeneration": ("gemma3n_mm", "Gemma3nForConditionalGeneration"),    # noqa: E501
242
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
243
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),  # noqa: E501
Jee Jee Li's avatar
Jee Jee Li committed
244
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),  # noqa: E501
245
    "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"),  # noqa: E501
246
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
247
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
248
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
Lyu Han's avatar
Lyu Han committed
249
    "InternS1ForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
250
    "InternVLForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
251
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
252
    "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"),  # noqa: E501
253
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
254
    "KeyeVL1_5ForConditionalGeneration": ("keye_vl1_5", "KeyeVL1_5ForConditionalGeneration"), # noqa: E501
255
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
256
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
257
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
258
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
259
260
261
262
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
263
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
264
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
265
    "MiniMaxVL01ForConditionalGeneration": ("minimax_vl_01", "MiniMaxVL01ForConditionalGeneration"),  # noqa: E501
266
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
267
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
268
    "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"),  # noqa: E501
269
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
270
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
271
    "Ovis": ("ovis", "Ovis"),
272
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
273
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
274
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
275
276
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
    "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"),  # noqa: E501
277
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
278
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
279
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
280
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
281
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
282
    "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
283
    "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
284
285
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),  # noqa: E501
    "Qwen3VLMoeForConditionalGeneration": ("qwen3_vl_moe", "Qwen3VLMoeForConditionalGeneration"),  # noqa: E501
286
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
287
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
288
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
289
    "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"),  # noqa: E501
290
    "UltravoxModel": ("ultravox", "UltravoxModel"),
Patrick von Platen's avatar
Patrick von Platen committed
291
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
292
    # [Encoder-decoder]
293
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
294
}
295
296

_SPECULATIVE_DECODING_MODELS = {
297
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
298
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
299
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
300
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
301
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
302
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
303
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
304
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
305
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
306
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
XuruiYang's avatar
XuruiYang committed
307
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
308
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
309
    "MedusaModel": ("medusa", "Medusa"),
310
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
311
312
313
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
314
}
315

316
_TRANSFORMERS_SUPPORTED_MODELS = {
317
318
319
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
320
321
322
323
    "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
}

_TRANSFORMERS_BACKEND_MODELS = {
324
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
325
    "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
326
327
328
329
330
331
    "TransformersMoEForCausalLM": ("transformers_moe", "TransformersMoEForCausalLM"),  # noqa: E501
    "TransformersMoEForMultimodalLM": ("transformers_moe", "TransformersMoEForMultimodalLM"),  # noqa: E501
    "TransformersEmbeddingModel": ("transformers_pooling", "TransformersEmbeddingModel"),  # noqa: E501
    "TransformersForSequenceClassification": ("transformers_pooling", "TransformersForSequenceClassification"),  # noqa: E501
    "TransformersMoEForSequenceClassification": ("transformers_pooling", "TransformersMoEForSequenceClassification"),  # noqa: E501
    "TransformersMoEEmbeddingModel": ("transformers_pooling", "TransformersMoEEmbeddingModel"),  # noqa: E501
332
}
333
# yapf: enable
334

335
_VLLM_MODELS = {
336
    **_TEXT_GENERATION_MODELS,
337
    **_EMBEDDING_MODELS,
338
    **_CROSS_ENCODER_MODELS,
339
    **_MULTIMODAL_MODELS,
340
    **_SPECULATIVE_DECODING_MODELS,
341
342
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
343
344
}

345
346
347
348
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
349
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
350

351
_PREVIOUSLY_SUPPORTED_MODELS = {
352
    "MotifForCausalLM": "0.10.2",
353
    "Phi3SmallForCausalLM": "0.9.2",
354
    "Phi4FlashForCausalLM": "0.10.2",
355
356
357
358
359
360
361
362
363
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "BartModel": "0.10.2",
    "BartForConditionalGeneration": "0.10.2",
    "DonutForConditionalGeneration": "0.10.2",
    "Florence2ForConditionalGeneration": "0.10.2",
    "MBartForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
364

365

366
367
@dataclass(frozen=True)
class _ModelInfo:
368
    architecture: str
369
    is_text_generation_model: bool
370
    is_pooling_model: bool
371
    default_pooling_type: str
372
    supports_cross_encoding: bool
373
    supports_multimodal: bool
374
    supports_multimodal_raw_input_only: bool
375
    supports_multimodal_encoder_tp_data: bool
376
    supports_pp: bool
377
378
    has_inner_state: bool
    is_attention_free: bool
379
    is_hybrid: bool
380
    has_noops: bool
381
    supports_transcription: bool
382
    supports_transcription_only: bool
383
    supports_v0_only: bool
384
385

    @staticmethod
386
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
387
        return _ModelInfo(
388
            architecture=model.__name__,
389
            is_text_generation_model=is_text_generation_model(model),
390
            is_pooling_model=is_pooling_model(model),
391
            default_pooling_type=get_default_pooling_type(model),
392
            supports_cross_encoding=supports_cross_encoding(model),
393
            supports_multimodal=supports_multimodal(model),
394
395
396
397
398
399
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
400
            supports_pp=supports_pp(model),
401
402
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
403
            is_hybrid=is_hybrid(model),
404
            supports_transcription=supports_transcription(model),
405
406
407
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
408
            supports_v0_only=supports_v0_only(model),
409
            has_noops=has_noops(model),
410
        )
411
412


413
414
415
416
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
417

418
    @abstractmethod
419
    def load_model_cls(self) -> type[nn.Module]:
420
        raise NotImplementedError
421
422


423
424
425
426
427
428
429
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
430
    model_cls: type[nn.Module]
431
432

    @staticmethod
433
    def from_model_cls(model_cls: type[nn.Module]):
434
435
436
437
438
439
440
441
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

442
    def load_model_cls(self) -> type[nn.Module]:
443
444
445
446
447
448
449
450
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
451

452
453
454
    module_name: str
    class_name: str

455
456
457
458
459
460
461
462
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

463
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
464
465
        try:
            try:
466
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
467
468
469
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
470
471
472
473
474
                logger.debug(
                    ("Cached model info file for class %s.%s not found"),
                    self.module_name,
                    self.class_name,
                )
475
476
477
                return None

            if mi_dict["hash"] != module_hash:
478
479
480
481
482
                logger.debug(
                    ("Cached model info file for class %s.%s is stale"),
                    self.module_name,
                    self.class_name,
                )
483
484
485
486
487
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
488
489
490
491
492
            logger.exception(
                ("Cached model info for class %s.%s error. "),
                self.module_name,
                self.class_name,
            )
493
494
            return None

495
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
496
497
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
498

499
500
501
502
503
504
505
506
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
507
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
508
509
510
511
512
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
513
    def inspect_model_cls(self) -> _ModelInfo:
514
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
515
        module_hash = None
516

517
518
519
520
521
522
        if model_path.exists():
            with open(model_path, "rb") as f:
                module_hash = hashlib.md5(f.read()).hexdigest()

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
523
524
525
526
527
                logger.debug(
                    ("Loaded model info for class %s.%s from cache"),
                    self.module_name,
                    self.class_name,
                )
528
529
                return mi
            else:
530
531
532
533
534
                logger.debug(
                    ("Cache model info for class %s.%s miss. Loading model instead."),
                    self.module_name,
                    self.class_name,
                )
535
536
537

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
538
539
540
541
542
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
543
544

        # save cache file
545
546
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
547
548

        return mi
549

550
    def load_model_cls(self) -> type[nn.Module]:
551
552
553
554
555
556
557
558
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
559
) -> Optional[type[nn.Module]]:
560
    from vllm.platforms import current_platform
561

562
    current_platform.verify_model_arch(model_arch)
563
564
565
    try:
        return model.load_model_cls()
    except Exception:
566
        logger.exception("Error in loading model architecture '%s'", model_arch)
567
        return None
568
569


570
571
572
573
574
575
576
577
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
578
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
579
        return None
580
581


582
583
584
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
585
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
586

587
    def get_supported_archs(self) -> Set[str]:
588
        return self.models.keys()
589

590
591
592
    def register_model(
        self,
        model_arch: str,
593
        model_cls: Union[type[nn.Module], str],
594
    ) -> None:
595
596
597
        """
        Register an external model to be used in vLLM.

598
        `model_cls` can be either:
599

600
        - A [`torch.nn.Module`][] class directly referencing the model.
601
        - A string in the format `<module>:<class>` which can be used to
602
603
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
604
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
605
        """
606
607
608
609
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

610
        if model_arch in self.models:
611
612
            logger.warning(
                "Model architecture %s is already registered, and will be "
613
614
615
616
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
617
618
619
620
621
622

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
623

624
            model = _LazyRegisteredModel(*split_str)
625
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
626
            model = _RegisteredModel.from_model_cls(model_cls)
627
        else:
628
629
630
631
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
632
            raise TypeError(msg)
633

634
        self.models[model_arch] = model
635

636
    def _raise_for_unsupported(self, architectures: list[str]):
637
        all_supported_archs = self.get_supported_archs()
638

639
640
641
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
642
643
                "to be inspected. Please check the logs for more details."
            )
644

645
646
647
648
649
650
651
652
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
653
654
                    "use this model architecture."
                )
655

656
657
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
658
659
            f"Supported architectures: {all_supported_archs}"
        )
660

661
    def _try_load_model_cls(self, model_arch: str) -> Optional[type[nn.Module]]:
662
663
        if model_arch not in self.models:
            return None
664

665
        return _try_load_model_cls(model_arch, self.models[model_arch])
666

667
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
668
669
        if model_arch not in self.models:
            return None
670

671
672
673
674
675
676
677
678
679
680
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> Optional[str]:
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

681
682
683
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
717
                if model_config.model_impl != "transformers":
718
719
720
721
722
723
724
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
725
726
                    "'auto_map' (relevant if the model is custom)."
                )
727
728

        if not model_module.is_backend_compatible():
729
            if model_config.model_impl != "transformers":
730
                return None
731

732
733
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
734
735
                "is not compatible with vLLM."
            )
736

737
        return model_config._get_transformers_backend_cls()
738

739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
764

765
766
    def inspect_model_cls(
        self,
767
        architectures: Union[str, list[str]],
768
        model_config: ModelConfig,
769
    ) -> tuple[_ModelInfo, str]:
770
771
        if isinstance(architectures, str):
            architectures = [architectures]
772
773
        if not architectures:
            raise ValueError("No model architectures are specified")
774
775

        # Require transformers impl
776
        if model_config.model_impl == "transformers":
777
            arch = self._try_resolve_transformers(architectures[0], model_config)
778
779
780
781
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
782
        elif model_config.model_impl == "terratorch":
783
784
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
785

786
        # Fallback to transformers impl (after resolving convert_type)
787
788
789
790
791
792
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
793
794
795
796
797
798
799
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
800
            model_info = self._try_inspect_model_cls(normalized_arch)
801
            if model_info is not None:
802
                return (model_info, arch)
803

804
        # Fallback to transformers impl (before resolving runner_type)
805
806
807
808
809
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
810
811
812
813
814
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

815
        return self._raise_for_unsupported(architectures)
816

817
818
    def resolve_model_cls(
        self,
819
        architectures: Union[str, list[str]],
820
        model_config: ModelConfig,
821
    ) -> tuple[type[nn.Module], str]:
822
823
        if isinstance(architectures, str):
            architectures = [architectures]
824
825
        if not architectures:
            raise ValueError("No model architectures are specified")
826
827

        # Require transformers impl
828
        if model_config.model_impl == "transformers":
829
            arch = self._try_resolve_transformers(architectures[0], model_config)
830
831
832
833
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
834
        elif model_config.model_impl == "terratorch":
835
836
837
838
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
839

840
        # Fallback to transformers impl (after resolving convert_type)
841
842
843
844
845
846
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
847
848
849
850
851
852
853
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
854
            model_cls = self._try_load_model_cls(normalized_arch)
855
856
            if model_cls is not None:
                return (model_cls, arch)
857

858
        # Fallback to transformers impl (before resolving runner_type)
859
860
861
862
863
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
864
865
866
867
868
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

869
        return self._raise_for_unsupported(architectures)
870

871
872
    def is_text_generation_model(
        self,
873
        architectures: Union[str, list[str]],
874
        model_config: ModelConfig,
875
    ) -> bool:
876
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
877
        return model_cls.is_text_generation_model
878

879
    def is_pooling_model(
880
        self,
881
        architectures: Union[str, list[str]],
882
        model_config: ModelConfig,
883
    ) -> bool:
884
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
885
        return model_cls.is_pooling_model
886

887
888
    def is_cross_encoder_model(
        self,
889
        architectures: Union[str, list[str]],
890
        model_config: ModelConfig,
891
    ) -> bool:
892
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
893
        return model_cls.supports_cross_encoding
894

895
896
    def is_multimodal_model(
        self,
897
        architectures: Union[str, list[str]],
898
        model_config: ModelConfig,
899
    ) -> bool:
900
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
901
        return model_cls.supports_multimodal
902

903
    def is_multimodal_raw_input_only_model(
904
905
        self,
        architectures: Union[str, list[str]],
906
        model_config: ModelConfig,
907
    ) -> bool:
908
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
909
        return model_cls.supports_multimodal_raw_input_only
910

911
912
    def is_pp_supported_model(
        self,
913
        architectures: Union[str, list[str]],
914
        model_config: ModelConfig,
915
    ) -> bool:
916
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
917
        return model_cls.supports_pp
918

919
920
    def model_has_inner_state(
        self,
921
        architectures: Union[str, list[str]],
922
        model_config: ModelConfig,
923
    ) -> bool:
924
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
925
        return model_cls.has_inner_state
926

927
928
    def is_attention_free_model(
        self,
929
        architectures: Union[str, list[str]],
930
        model_config: ModelConfig,
931
    ) -> bool:
932
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
933
        return model_cls.is_attention_free
934

935
936
    def is_hybrid_model(
        self,
937
        architectures: Union[str, list[str]],
938
        model_config: ModelConfig,
939
    ) -> bool:
940
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
941
942
        return model_cls.is_hybrid

943
944
    def is_noops_model(
        self,
945
        architectures: Union[str, list[str]],
946
        model_config: ModelConfig,
947
    ) -> bool:
948
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
949
950
        return model_cls.has_noops

951
952
    def is_transcription_model(
        self,
953
        architectures: Union[str, list[str]],
954
        model_config: ModelConfig,
955
    ) -> bool:
956
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
957
958
        return model_cls.supports_transcription

959
960
961
    def is_transcription_only_model(
        self,
        architectures: Union[str, list[str]],
962
        model_config: ModelConfig,
963
    ) -> bool:
964
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
965
966
        return model_cls.supports_transcription_only

967
968
    def is_v1_compatible(
        self,
969
        architectures: Union[str, list[str]],
970
        model_config: ModelConfig,
971
    ) -> bool:
972
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
973
974
        return not model_cls.supports_v0_only

975

976
977
978
979
980
981
982
983
984
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
985
986
987
988
989

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
990
991
992
993
994
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

995
        # `cloudpickle` allows pickling lambda functions directly
996
        import cloudpickle
997

998
        input_bytes = cloudpickle.dumps((fn, output_filepath))
999
1000
1001

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1002
1003
1004
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1005
1006
1007
1008
1009
1010

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1011
1012
1013
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1014

1015
        with open(output_filepath, "rb") as f:
1016
1017
1018
1019
1020
1021
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1022

1023
1024
1025
1026
1027
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1028
1029
1030

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1031
1032
1033


if __name__ == "__main__":
1034
    _run()