registry.py 46.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
34
from vllm.utils.hashing import safe_hash
35

36
37
38
39
40
41
42
43
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
    from vllm.config.pooler import PoolingTypeStr
else:
    AttnTypeStr = Any
    PoolingTypeStr = Any


44
45
46
47
48
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
49
    requires_raw_input_tokens,
50
    supports_cross_encoding,
51
    supports_mamba_prefix_caching,
52
53
54
55
56
57
58
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
59
    get_attn_type,
60
61
62
63
    get_default_pooling_type,
    is_pooling_model,
    is_text_generation_model,
)
64
65
66

logger = init_logger(__name__)

67
68
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
69
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
70
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
71
72
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
73
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
74
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
75
76
77
78
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
79
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
80
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
81
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
82
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
83
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
84
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
85
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
86
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
87
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
88
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
89
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
90
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
91
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
92
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
93
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
94
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
95
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
96
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
97
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
98
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
99
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
100
101
102
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
103
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
104
105
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
106
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
107
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
108
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
109
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
110
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
111
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
112
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
113
114
115
116
117
118
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
119
120
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),  # noqa: E501
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),  # noqa: E501
121
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
122
123
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
124
125
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
126
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
127
128
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
129
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
130
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
131
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
132
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
133
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
134
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),  # noqa: E501
135
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
136
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
137
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
138
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
139
140
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
141
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
142
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
143
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
144
145
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
146
147
148
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
149
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
150
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
151
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
152
153
154
155
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
156
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
157
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
158
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
159
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
160
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
161
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
162
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
163
164
165
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
166
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
167
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
168
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
169
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
170
171
172
173
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
174
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
175
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
176
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
177
178
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
179
180
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
181
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
182
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Song's avatar
Song committed
183
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
184
185
186
187
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
188
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
189
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
190
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
191
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
192
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
193
194
195
}

_EMBEDDING_MODELS = {
196
    # [Text-only]
197
    "BertModel": ("bert", "BertEmbeddingModel"),
198
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
199
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
200
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
201
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
202
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
203
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
204
    "GritLM": ("gritlm", "GritLM"),
205
206
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
207
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
208
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
209
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
210
    "LlamaModel": ("llama", "LlamaForCausalLM"),
211
212
    **{
        # Multiple models share the same architecture, so we include them all
213
214
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
215
216
        if arch == "LlamaForCausalLM"
    },
217
    "MistralModel": ("llama", "LlamaForCausalLM"),
218
    "ModernBertModel": ("modernbert", "ModernBertModel"),
219
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
220
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
221
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
222
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
223
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
224
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
225
226
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
227
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
228
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
229
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
230
    # [Multimodal]
231
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
232
233
234
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
235
    ),
236
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
237
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
238
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
239
240
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
241
    # models for the time being.
242
243
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
244
245
}

246
247
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
248
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
249
250
251
252
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
253
254
255
256
257
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
    ),
258
259
260
261
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
262
263
264
265
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
266
267
268
269
270
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
271
272
}

273
_MULTIMODAL_MODELS = {
274
    # [Decoder-only]
275
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
276
277
278
279
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
280
281
282
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
283
    ),
284
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
285
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
286
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
287
288
289
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
290
    ),
291
292
293
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
294
    ),
295
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
296
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
Roger Wang's avatar
Roger Wang committed
297
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
298
299
300
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
301
    ),
302
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
303
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
304
305
306
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
307
    ),
308
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
309
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
310
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),  # noqa: E501
Jee Jee Li's avatar
Jee Jee Li committed
311
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),  # noqa: E501
312
313
314
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
315
    ),
316
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
317
318
319
320
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
321
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
322
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
Zero's avatar
Zero committed
323
324
325
326
    "OpenCUAForConditionalGeneration": (
        "opencua",
        "OpenCUAForConditionalGeneration",
    ),
327
328
329
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
330
    ),
331
332
333
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
334
    ),
335
336
337
338
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
339
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
340
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),  # noqa: E501
341
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
342
343
344
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
345
    ),
346
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
347
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
348
349
350
351
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
352
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
353
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
354
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
355
356
357
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
358
    ),
359
360
361
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
362
    ),
363
364
365
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
366
    ),
367
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
368
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
369
370
371
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
372
    ),
373
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
374
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
375
376
377
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
378
    ),
379
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
380
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
381
    "Ovis": ("ovis", "Ovis"),
382
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
383
384
385
386
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
387
388
389
390
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
391
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
392
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
393
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
394
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
395
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
396
397
398
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
399
    ),
400
401
402
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
403
    ),
404
405
406
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
407
    ),
408
409
410
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
411
    ),
412
413
414
415
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
416
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),  # noqa: E501
417
418
419
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
420
    ),
421
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
422
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
423
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
424
425
426
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
427
    ),
428
    "UltravoxModel": ("ultravox", "UltravoxModel"),
Patrick von Platen's avatar
Patrick von Platen committed
429
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
Patrick von Platen's avatar
Patrick von Platen committed
430
    "VoxtralStreamingGeneration": ("voxtral_streaming", "VoxtralStreamingGeneration"),  # noqa: E501
431
    # [Encoder-decoder]
432
433
434
435
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
436
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
437
}
438
439

_SPECULATIVE_DECODING_MODELS = {
440
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
441
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
442
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
443
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
444
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
445
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
446
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
447
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
448
449
450
451
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
452
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
453
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
454
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
XuruiYang's avatar
XuruiYang committed
455
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
456
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
457
    "MedusaModel": ("medusa", "Medusa"),
458
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
459
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
460
461
462
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
463
}
464

465
_TRANSFORMERS_SUPPORTED_MODELS = {
466
467
468
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
469
470
471
472
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
473
474
475
}

_TRANSFORMERS_BACKEND_MODELS = {
476
    # Text generation models
477
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
496
    "TransformersForSequenceClassification": (
497
        "transformers",
498
        "TransformersForSequenceClassification",
499
    ),
500
    "TransformersMoEForSequenceClassification": (
501
        "transformers",
502
        "TransformersMoEForSequenceClassification",
503
    ),
504
505
506
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
507
    ),
508
}
509

510
_VLLM_MODELS = {
511
    **_TEXT_GENERATION_MODELS,
512
    **_EMBEDDING_MODELS,
513
    **_CROSS_ENCODER_MODELS,
514
    **_MULTIMODAL_MODELS,
515
    **_SPECULATIVE_DECODING_MODELS,
516
517
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
518
519
}

520
521
522
523
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
524
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
525

526
_PREVIOUSLY_SUPPORTED_MODELS = {
527
    "MotifForCausalLM": "0.10.2",
528
    "Phi3SmallForCausalLM": "0.9.2",
529
    "Phi4FlashForCausalLM": "0.10.2",
530
    "Phi4MultimodalForCausalLM": "0.12.0",
531
532
533
534
535
536
537
538
539
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "BartModel": "0.10.2",
    "BartForConditionalGeneration": "0.10.2",
    "DonutForConditionalGeneration": "0.10.2",
    "Florence2ForConditionalGeneration": "0.10.2",
    "MBartForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
540

541

542
543
@dataclass(frozen=True)
class _ModelInfo:
544
    architecture: str
545
    is_text_generation_model: bool
546
    is_pooling_model: bool
547
548
    attn_type: AttnTypeStr
    default_pooling_type: PoolingTypeStr
549
    supports_cross_encoding: bool
550
    supports_multimodal: bool
551
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
552
    requires_raw_input_tokens: bool
553
    supports_multimodal_encoder_tp_data: bool
554
    supports_pp: bool
555
556
    has_inner_state: bool
    is_attention_free: bool
557
    is_hybrid: bool
558
    has_noops: bool
559
    supports_mamba_prefix_caching: bool
560
    supports_transcription: bool
561
    supports_transcription_only: bool
562
563

    @staticmethod
564
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
565
        return _ModelInfo(
566
            architecture=model.__name__,
567
            is_text_generation_model=is_text_generation_model(model),
568
            is_pooling_model=is_pooling_model(model),
569
            default_pooling_type=get_default_pooling_type(model),
570
            attn_type=get_attn_type(model),
571
            supports_cross_encoding=supports_cross_encoding(model),
572
            supports_multimodal=supports_multimodal(model),
573
574
575
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
576
            requires_raw_input_tokens=requires_raw_input_tokens(model),
577
578
579
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
580
            supports_pp=supports_pp(model),
581
582
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
583
            is_hybrid=is_hybrid(model),
584
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
585
            supports_transcription=supports_transcription(model),
586
587
588
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
589
            has_noops=has_noops(model),
590
        )
591
592


593
594
595
596
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
597

598
    @abstractmethod
599
    def load_model_cls(self) -> type[nn.Module]:
600
        raise NotImplementedError
601
602


603
604
605
606
607
608
609
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
610
    model_cls: type[nn.Module]
611
612

    @staticmethod
613
    def from_model_cls(model_cls: type[nn.Module]):
614
615
616
617
618
619
620
621
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

622
    def load_model_cls(self) -> type[nn.Module]:
623
624
625
626
627
628
629
630
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
631

632
633
634
    module_name: str
    class_name: str

635
636
637
638
639
640
641
642
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

643
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
644
645
        try:
            try:
646
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
647
648
649
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
650
                logger.debug(
651
                    "Cached model info file for class %s.%s not found",
652
653
654
                    self.module_name,
                    self.class_name,
                )
655
656
657
                return None

            if mi_dict["hash"] != module_hash:
658
                logger.debug(
659
                    "Cached model info file for class %s.%s is stale",
660
661
662
                    self.module_name,
                    self.class_name,
                )
663
664
665
666
667
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
668
            logger.debug(
669
                "Cached model info for class %s.%s error. ",
670
671
672
                self.module_name,
                self.class_name,
            )
673
674
            return None

675
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
676
677
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
678

679
680
681
682
683
684
685
686
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
687
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
688
689
690
691
692
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
693
    def inspect_model_cls(self) -> _ModelInfo:
694
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
695
        module_hash = None
696

697
698
        if model_path.exists():
            with open(model_path, "rb") as f:
699
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
700
701
702

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
703
                logger.debug(
704
                    "Loaded model info for class %s.%s from cache",
705
706
707
                    self.module_name,
                    self.class_name,
                )
708
709
                return mi
            else:
710
                logger.debug(
711
                    "Cache model info for class %s.%s miss. Loading model instead.",
712
713
714
                    self.module_name,
                    self.class_name,
                )
715
716
717

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
718
719
720
721
722
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
723
724

        # save cache file
725
726
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
727
728

        return mi
729

730
    def load_model_cls(self) -> type[nn.Module]:
731
732
733
734
735
736
737
738
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
739
) -> type[nn.Module] | None:
740
    from vllm.platforms import current_platform
741

742
    current_platform.verify_model_arch(model_arch)
743
744
745
    try:
        return model.load_model_cls()
    except Exception:
746
        logger.exception("Error in loading model architecture '%s'", model_arch)
747
        return None
748
749


750
751
752
753
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
754
) -> _ModelInfo | None:
755
756
757
    try:
        return model.inspect_model_cls()
    except Exception:
758
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
759
        return None
760
761


762
763
764
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
765
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
766

767
    def get_supported_archs(self) -> Set[str]:
768
        return self.models.keys()
769

770
771
772
    def register_model(
        self,
        model_arch: str,
773
        model_cls: type[nn.Module] | str,
774
    ) -> None:
775
776
777
        """
        Register an external model to be used in vLLM.

778
        `model_cls` can be either:
779

780
        - A [`torch.nn.Module`][] class directly referencing the model.
781
        - A string in the format `<module>:<class>` which can be used to
782
783
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
784
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
785
        """
786
787
788
789
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

790
        if model_arch in self.models:
791
792
            logger.warning(
                "Model architecture %s is already registered, and will be "
793
794
795
796
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
797
798
799
800
801
802

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
803

804
            model = _LazyRegisteredModel(*split_str)
805
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
806
            model = _RegisteredModel.from_model_cls(model_cls)
807
        else:
808
809
810
811
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
812
            raise TypeError(msg)
813

814
        self.models[model_arch] = model
815

816
    def _raise_for_unsupported(self, architectures: list[str]):
817
        all_supported_archs = self.get_supported_archs()
818

819
820
821
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
822
823
                "to be inspected. Please check the logs for more details."
            )
824

825
826
827
828
829
830
831
832
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
833
834
                    "use this model architecture."
                )
835

836
837
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
838
839
            f"Supported architectures: {all_supported_archs}"
        )
840

841
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
842
843
        if model_arch not in self.models:
            return None
844

845
        return _try_load_model_cls(model_arch, self.models[model_arch])
846

847
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
848
849
        if model_arch not in self.models:
            return None
850

851
852
853
854
855
856
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
857
    ) -> str | None:
858
859
860
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

861
862
863
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
897
                if model_config.model_impl != "transformers":
898
899
900
901
902
903
904
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
905
906
                    "'auto_map' (relevant if the model is custom)."
                )
907
908

        if not model_module.is_backend_compatible():
909
            if model_config.model_impl != "transformers":
910
                return None
911

912
913
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
914
915
                "is not compatible with vLLM."
            )
916

917
        return model_config._get_transformers_backend_cls()
918

919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
944

945
946
    def inspect_model_cls(
        self,
947
        architectures: str | list[str],
948
        model_config: ModelConfig,
949
    ) -> tuple[_ModelInfo, str]:
950
951
        if isinstance(architectures, str):
            architectures = [architectures]
952
953
        if not architectures:
            raise ValueError("No model architectures are specified")
954
955

        # Require transformers impl
956
        if model_config.model_impl == "transformers":
957
            arch = self._try_resolve_transformers(architectures[0], model_config)
958
959
960
961
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
962
        elif model_config.model_impl == "terratorch":
963
964
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
965

966
        # Fallback to transformers impl (after resolving convert_type)
967
968
969
970
971
972
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
973
974
975
976
977
978
979
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
980
            model_info = self._try_inspect_model_cls(normalized_arch)
981
            if model_info is not None:
982
                return (model_info, arch)
983

984
        # Fallback to transformers impl (before resolving runner_type)
985
986
987
988
989
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
990
991
992
993
994
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

995
        return self._raise_for_unsupported(architectures)
996

997
998
    def resolve_model_cls(
        self,
999
        architectures: str | list[str],
1000
        model_config: ModelConfig,
1001
    ) -> tuple[type[nn.Module], str]:
1002
1003
        if isinstance(architectures, str):
            architectures = [architectures]
1004
1005
        if not architectures:
            raise ValueError("No model architectures are specified")
1006
1007

        # Require transformers impl
1008
        if model_config.model_impl == "transformers":
1009
            arch = self._try_resolve_transformers(architectures[0], model_config)
1010
1011
1012
1013
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1014
        elif model_config.model_impl == "terratorch":
1015
1016
1017
1018
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1019

1020
        # Fallback to transformers impl (after resolving convert_type)
1021
1022
1023
1024
1025
1026
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1027
1028
1029
1030
1031
1032
1033
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1034
            model_cls = self._try_load_model_cls(normalized_arch)
1035
1036
            if model_cls is not None:
                return (model_cls, arch)
1037

1038
        # Fallback to transformers impl (before resolving runner_type)
1039
1040
1041
1042
1043
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1044
1045
1046
1047
1048
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1049
        return self._raise_for_unsupported(architectures)
1050

1051
1052
    def is_text_generation_model(
        self,
1053
        architectures: str | list[str],
1054
        model_config: ModelConfig,
1055
    ) -> bool:
1056
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1057
        return model_cls.is_text_generation_model
1058

1059
    def is_pooling_model(
1060
        self,
1061
        architectures: str | list[str],
1062
        model_config: ModelConfig,
1063
    ) -> bool:
1064
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1065
        return model_cls.is_pooling_model
1066

1067
1068
    def is_cross_encoder_model(
        self,
1069
        architectures: str | list[str],
1070
        model_config: ModelConfig,
1071
    ) -> bool:
1072
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1073
        return model_cls.supports_cross_encoding
1074

1075
1076
    def is_multimodal_model(
        self,
1077
        architectures: str | list[str],
1078
        model_config: ModelConfig,
1079
    ) -> bool:
1080
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1081
        return model_cls.supports_multimodal
1082

1083
    def is_multimodal_raw_input_only_model(
1084
        self,
1085
        architectures: str | list[str],
1086
        model_config: ModelConfig,
1087
    ) -> bool:
1088
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1089
        return model_cls.supports_multimodal_raw_input_only
1090

1091
1092
    def is_pp_supported_model(
        self,
1093
        architectures: str | list[str],
1094
        model_config: ModelConfig,
1095
    ) -> bool:
1096
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1097
        return model_cls.supports_pp
1098

1099
1100
    def model_has_inner_state(
        self,
1101
        architectures: str | list[str],
1102
        model_config: ModelConfig,
1103
    ) -> bool:
1104
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1105
        return model_cls.has_inner_state
1106

1107
1108
    def is_attention_free_model(
        self,
1109
        architectures: str | list[str],
1110
        model_config: ModelConfig,
1111
    ) -> bool:
1112
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1113
        return model_cls.is_attention_free
1114

1115
1116
    def is_hybrid_model(
        self,
1117
        architectures: str | list[str],
1118
        model_config: ModelConfig,
1119
    ) -> bool:
1120
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1121
1122
        return model_cls.is_hybrid

1123
1124
    def is_noops_model(
        self,
1125
        architectures: str | list[str],
1126
        model_config: ModelConfig,
1127
    ) -> bool:
1128
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1129
1130
        return model_cls.has_noops

1131
1132
    def is_transcription_model(
        self,
1133
        architectures: str | list[str],
1134
        model_config: ModelConfig,
1135
    ) -> bool:
1136
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1137
1138
        return model_cls.supports_transcription

1139
1140
    def is_transcription_only_model(
        self,
1141
        architectures: str | list[str],
1142
        model_config: ModelConfig,
1143
    ) -> bool:
1144
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1145
1146
        return model_cls.supports_transcription_only

1147

1148
1149
1150
1151
1152
1153
1154
1155
1156
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1157
1158
1159
1160
1161

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1162
1163
1164
1165
1166
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1167
        # `cloudpickle` allows pickling lambda functions directly
1168
        import cloudpickle
1169

1170
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1171
1172
1173

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1174
1175
1176
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1177
1178
1179
1180
1181
1182

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1183
1184
1185
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1186

1187
        with open(output_filepath, "rb") as f:
1188
1189
1190
1191
1192
1193
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1194

1195
1196
1197
1198
1199
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1200
1201
1202

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1203
1204
1205


if __name__ == "__main__":
1206
    _run()