registry.py 50.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
34
from vllm.utils.hashing import safe_hash
35

36
37
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
38
    from vllm.config.pooler import SequencePoolingType, TokenPoolingType
39
40
else:
    AttnTypeStr = Any
41
42
    SequencePoolingType = Any
    TokenPoolingType = Any
43
44


45
46
47
48
49
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
50
    requires_raw_input_tokens,
51
    supports_cross_encoding,
52
    supports_late_interaction,
53
    supports_mamba_prefix_caching,
54
55
56
57
58
59
60
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
61
    get_attn_type,
62
63
    get_default_seq_pooling_type,
    get_default_tok_pooling_type,
64
65
66
    is_pooling_model,
    is_text_generation_model,
)
67
68
69

logger = init_logger(__name__)

70
71
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
72
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
73
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
74
75
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
76
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
77
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
78
79
80
81
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
82
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
83
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
84
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
85
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
86
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
87
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
88
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
89
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
90
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
91
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
92
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
93
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
94
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
95
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
96
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
97
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
98
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
99
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
100
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
101
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
Kyungmin Lee's avatar
Kyungmin Lee committed
102
    "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
103
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
104
105
106
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
107
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
108
109
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
110
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
111
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
112
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
113
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
114
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
115
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
116
    "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
Jee Jee Li's avatar
Jee Jee Li committed
117
    "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
118
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
119
120
121
122
123
124
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
125
126
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),  # noqa: E501
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),  # noqa: E501
127
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
128
129
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
130
131
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
132
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
133
134
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
135
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
136
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
137
138
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
139
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
140
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
141
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
142
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),  # noqa: E501
143
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
144
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
145
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
146
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
147
148
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
149
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
150
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
151
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
152
153
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
154
155
156
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
157
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
158
    "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
159
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
160
161
162
163
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
164
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
165
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
166
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
167
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
168
    "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
169
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
170
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
171
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
172
173
174
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
175
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
176
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
177
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
178
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
179
180
181
182
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
183
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
184
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
185
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
186
187
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
188
189
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
190
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
191
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Li Xie's avatar
Li Xie committed
192
    "Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
Song's avatar
Song committed
193
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
csy0225's avatar
csy0225 committed
194
    "Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
195
196
197
198
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
199
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
200
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
201
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
202
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
203
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
204
205
206
}

_EMBEDDING_MODELS = {
207
    # [Text-only]
208
    "BertModel": ("bert", "BertEmbeddingModel"),
209
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
210
    "HF_ColBERT": ("colbert", "ColBERTModel"),
211
212
    "ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
    "ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
213
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
214
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
215
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
216
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
217
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
218
    "GritLM": ("gritlm", "GritLM"),
219
220
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
221
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
222
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
223
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
224
    "LlamaModel": ("llama", "LlamaForCausalLM"),
225
226
    **{
        # Multiple models share the same architecture, so we include them all
227
228
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
229
230
        if arch == "LlamaForCausalLM"
    },
231
    "MistralModel": ("llama", "LlamaForCausalLM"),
232
    "ModernBertModel": ("modernbert", "ModernBertModel"),
233
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
234
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
235
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
236
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
237
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
238
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
239
240
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
241
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
242
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
chengchengpei's avatar
chengchengpei committed
243
244
245
246
    "VoyageQwen3BidirectionalEmbedModel": (
        "voyage",
        "VoyageQwen3BidirectionalEmbedModel",
    ),
247
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
248
    "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
249
    # [Multimodal]
250
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
251
252
253
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
254
    ),
255
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
256
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
257
258
    "ColQwen3": ("colqwen3", "ColQwen3Model"),
    "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
259
    "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
260
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
261
262
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
263
    # models for the time being.
264
265
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
266
267
}

268
269
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
270
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
271
272
273
274
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
275
276
277
278
279
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
    ),
280
281
282
283
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
284
285
286
287
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
288
289
290
291
292
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
293
294
}

295
_MULTIMODAL_MODELS = {
296
    # [Decoder-only]
297
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
298
299
300
301
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
302
303
304
305
    "MusicFlamingoForConditionalGeneration": (
        "musicflamingo",
        "MusicFlamingoForConditionalGeneration",
    ),
306
307
308
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
309
    ),
310
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
311
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
312
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
313
314
315
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
316
    ),
317
318
319
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
320
    ),
321
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
322
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
RED's avatar
RED committed
323
    "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
324
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
325
326
327
328
    "Eagle2_5_VLForConditionalGeneration": (
        "eagle2_5_vl",
        "Eagle2_5_VLForConditionalGeneration",
    ),
329
330
331
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
332
    ),
333
    "FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),  # noqa: E501
334
335
336
337
    "FunAudioChatForConditionalGeneration": (
        "funaudiochat",
        "FunAudioChatForConditionalGeneration",
    ),
338
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
339
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
340
341
342
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
343
    ),
344
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
345
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
346
347
348
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
    "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),  # noqa: E501
349
350
351
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
352
    ),
353
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
354
355
356
357
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
ltd0924's avatar
ltd0924 committed
358
    "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
359
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
360
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
Zero's avatar
Zero committed
361
362
363
364
    "OpenCUAForConditionalGeneration": (
        "opencua",
        "OpenCUAForConditionalGeneration",
    ),
365
366
367
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
368
    ),
369
370
371
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
372
    ),
zxy's avatar
zxy committed
373
374
375
376
    "InternS1ProForConditionalGeneration": (
        "interns1_pro",
        "InternS1ProForConditionalGeneration",
    ),
377
378
379
380
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
381
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
382
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),  # noqa: E501
383
    "KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
384
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
385
386
387
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
388
    ),
389
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
390
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
391
    "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),  # noqa: E501
392
393
394
395
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
396
    "Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
397
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
398
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
399
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
400
401
402
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
403
    ),
404
405
406
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
407
    ),
408
409
410
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
411
    ),
412
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
413
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
414
415
416
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
417
    ),
418
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
419
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
420
421
422
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
423
    ),
424
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
425
    "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
426
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
427
428
429
430
    "OpenPanguVLForConditionalGeneration": (
        "openpangu_vl",
        "OpenPanguVLForConditionalGeneration",
    ),
431
    "Ovis": ("ovis", "Ovis"),
432
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
433
434
    "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
    "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
435
436
437
438
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
439
440
441
442
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
443
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
444
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
445
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
446
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
447
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
448
449
450
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
451
    ),
452
453
454
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
455
    ),
456
457
458
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
459
    ),
460
461
462
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
463
    ),
464
465
466
467
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
Roger Wang's avatar
Roger Wang committed
468
469
470
471
    "Qwen3ASRForConditionalGeneration": (
        "qwen3_asr",
        "Qwen3ASRForConditionalGeneration",
    ),
472
473
474
475
    "Qwen3ASRRealtimeGeneration": (
        "qwen3_asr_realtime",
        "Qwen3ASRRealtimeGeneration",
    ),
476
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),  # noqa: E501
477
478
479
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
480
    ),
481
482
483
484
485
486
487
488
    "Qwen3_5ForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5ForConditionalGeneration",
    ),
    "Qwen3_5MoeForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5MoeForConditionalGeneration",
    ),
489
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
490
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
491
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
492
493
494
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
495
    ),
496
    "UltravoxModel": ("ultravox", "UltravoxModel"),
Patrick von Platen's avatar
Patrick von Platen committed
497
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
498
    "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),  # noqa: E501
499
    # [Encoder-decoder]
500
501
502
503
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
504
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
505
}
506
507

_SPECULATIVE_DECODING_MODELS = {
508
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
509
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
510
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
511
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
512
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
513
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
514
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
515
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
516
517
518
519
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
520
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
521
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
522
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
523
    "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
XuruiYang's avatar
XuruiYang committed
524
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
525
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
526
    "Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
527
    "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
528
    "MedusaModel": ("medusa", "Medusa"),
529
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
530
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
csy0225's avatar
csy0225 committed
531
    "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
532
533
    "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
    "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
534
535
536
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
537
}
538

539
_TRANSFORMERS_SUPPORTED_MODELS = {
540
541
542
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
543
544
545
546
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
547
548
549
}

_TRANSFORMERS_BACKEND_MODELS = {
550
    # Text generation models
551
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
570
    "TransformersForSequenceClassification": (
571
        "transformers",
572
        "TransformersForSequenceClassification",
573
    ),
574
    "TransformersMoEForSequenceClassification": (
575
        "transformers",
576
        "TransformersMoEForSequenceClassification",
577
    ),
578
579
580
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
581
    ),
582
}
583

584
_VLLM_MODELS = {
585
    **_TEXT_GENERATION_MODELS,
586
    **_EMBEDDING_MODELS,
587
    **_CROSS_ENCODER_MODELS,
588
    **_MULTIMODAL_MODELS,
589
    **_SPECULATIVE_DECODING_MODELS,
590
591
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
592
593
}

594
595
596
597
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
598
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
599

600
_PREVIOUSLY_SUPPORTED_MODELS = {
601
    "MotifForCausalLM": "0.10.2",
602
    "Phi3SmallForCausalLM": "0.9.2",
603
    "Phi4FlashForCausalLM": "0.10.2",
604
    "Phi4MultimodalForCausalLM": "0.12.0",
605
606
607
608
609
610
611
612
613
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "BartModel": "0.10.2",
    "BartForConditionalGeneration": "0.10.2",
    "DonutForConditionalGeneration": "0.10.2",
    "Florence2ForConditionalGeneration": "0.10.2",
    "MBartForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
614

615

616
617
@dataclass(frozen=True)
class _ModelInfo:
618
    architecture: str
619
    is_text_generation_model: bool
620
    is_pooling_model: bool
621
    attn_type: AttnTypeStr
622
623
    default_seq_pooling_type: SequencePoolingType
    default_tok_pooling_type: TokenPoolingType
624
    supports_cross_encoding: bool
625
    supports_late_interaction: bool
626
    supports_multimodal: bool
627
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
628
    requires_raw_input_tokens: bool
629
    supports_multimodal_encoder_tp_data: bool
630
    supports_pp: bool
631
632
    has_inner_state: bool
    is_attention_free: bool
633
    is_hybrid: bool
634
    has_noops: bool
635
    supports_mamba_prefix_caching: bool
636
    supports_transcription: bool
637
    supports_transcription_only: bool
638
639

    @staticmethod
640
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
641
        return _ModelInfo(
642
            architecture=model.__name__,
643
            is_text_generation_model=is_text_generation_model(model),
644
            is_pooling_model=is_pooling_model(model),
645
646
            default_seq_pooling_type=get_default_seq_pooling_type(model),
            default_tok_pooling_type=get_default_tok_pooling_type(model),
647
            attn_type=get_attn_type(model),
648
            supports_cross_encoding=supports_cross_encoding(model),
649
            supports_late_interaction=supports_late_interaction(model),
650
            supports_multimodal=supports_multimodal(model),
651
652
653
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
654
            requires_raw_input_tokens=requires_raw_input_tokens(model),
655
656
657
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
658
            supports_pp=supports_pp(model),
659
660
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
661
            is_hybrid=is_hybrid(model),
662
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
663
            supports_transcription=supports_transcription(model),
664
665
666
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
667
            has_noops=has_noops(model),
668
        )
669
670


671
672
673
674
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
675

676
    @abstractmethod
677
    def load_model_cls(self) -> type[nn.Module]:
678
        raise NotImplementedError
679
680


681
682
683
684
685
686
687
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
688
    model_cls: type[nn.Module]
689
690

    @staticmethod
691
    def from_model_cls(model_cls: type[nn.Module]):
692
693
694
695
696
697
698
699
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

700
    def load_model_cls(self) -> type[nn.Module]:
701
702
703
704
705
706
707
708
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
709

710
711
712
    module_name: str
    class_name: str

713
714
715
716
717
718
719
720
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

721
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
722
723
        try:
            try:
724
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
725
726
727
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
728
                logger.debug(
729
                    "Cached model info file for class %s.%s not found",
730
731
732
                    self.module_name,
                    self.class_name,
                )
733
734
735
                return None

            if mi_dict["hash"] != module_hash:
736
                logger.debug(
737
                    "Cached model info file for class %s.%s is stale",
738
739
740
                    self.module_name,
                    self.class_name,
                )
741
742
743
744
745
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
746
            logger.debug(
747
                "Cached model info for class %s.%s error. ",
748
749
750
                self.module_name,
                self.class_name,
            )
751
752
            return None

753
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
754
755
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
756

757
758
759
760
761
762
763
764
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
765
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
766
767
768
769
770
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
771
    def inspect_model_cls(self) -> _ModelInfo:
772
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
773
        module_hash = None
774

775
776
        if model_path.exists():
            with open(model_path, "rb") as f:
777
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
778
779
780

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
781
                logger.debug(
782
                    "Loaded model info for class %s.%s from cache",
783
784
785
                    self.module_name,
                    self.class_name,
                )
786
787
                return mi
            else:
788
                logger.debug(
789
                    "Cache model info for class %s.%s miss. Loading model instead.",
790
791
792
                    self.module_name,
                    self.class_name,
                )
793
794
795

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
796
797
798
799
800
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
801
802

        # save cache file
803
804
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
805
806

        return mi
807

808
    def load_model_cls(self) -> type[nn.Module]:
809
810
811
812
813
814
815
816
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
817
) -> type[nn.Module] | None:
818
    from vllm.platforms import current_platform
819

820
    current_platform.verify_model_arch(model_arch)
821
822
823
    try:
        return model.load_model_cls()
    except Exception:
824
        logger.exception("Error in loading model architecture '%s'", model_arch)
825
        return None
826
827


828
829
830
831
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
832
) -> _ModelInfo | None:
833
834
835
    try:
        return model.inspect_model_cls()
    except Exception:
836
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
837
        return None
838
839


840
841
842
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
843
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
844

845
    def get_supported_archs(self) -> Set[str]:
846
        return self.models.keys()
847

848
849
850
    def register_model(
        self,
        model_arch: str,
851
        model_cls: type[nn.Module] | str,
852
    ) -> None:
853
854
855
        """
        Register an external model to be used in vLLM.

856
        `model_cls` can be either:
857

858
        - A [`torch.nn.Module`][] class directly referencing the model.
859
        - A string in the format `<module>:<class>` which can be used to
860
861
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
862
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
863
        """
864
865
866
867
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

868
        if model_arch in self.models:
869
870
            logger.warning(
                "Model architecture %s is already registered, and will be "
871
872
873
874
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
875
876
877
878
879
880

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
881

882
            model = _LazyRegisteredModel(*split_str)
883
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
884
            model = _RegisteredModel.from_model_cls(model_cls)
885
        else:
886
887
888
889
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
890
            raise TypeError(msg)
891

892
        self.models[model_arch] = model
893

894
    def _raise_for_unsupported(self, architectures: list[str]):
895
        all_supported_archs = self.get_supported_archs()
896

897
898
899
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
900
901
                "to be inspected. Please check the logs for more details."
            )
902

903
904
905
906
907
908
909
910
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
911
912
                    "use this model architecture."
                )
913

914
915
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
916
917
            f"Supported architectures: {all_supported_archs}"
        )
918

919
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
920
921
        if model_arch not in self.models:
            return None
922

923
        return _try_load_model_cls(model_arch, self.models[model_arch])
924

925
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
926
927
        if model_arch not in self.models:
            return None
928

929
930
931
932
933
934
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
935
    ) -> str | None:
936
937
938
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

939
940
941
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
958
                        trust_remote_code=model_config.trust_remote_code,
959
960
961
962
963
964
965
966
967
968
969
970
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
971
                        trust_remote_code=model_config.trust_remote_code,
972
973
974
975
976
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
977
                if model_config.model_impl != "transformers":
978
979
980
981
982
983
984
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
985
986
                    "'auto_map' (relevant if the model is custom)."
                )
987
988

        if not model_module.is_backend_compatible():
989
            if model_config.model_impl != "transformers":
990
                return None
991

992
993
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
994
995
                "is not compatible with vLLM."
            )
996

997
        return model_config._get_transformers_backend_cls()
998

999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
1024

1025
1026
    def inspect_model_cls(
        self,
1027
        architectures: str | list[str],
1028
        model_config: ModelConfig,
1029
    ) -> tuple[_ModelInfo, str]:
1030
1031
        if isinstance(architectures, str):
            architectures = [architectures]
1032
1033
        if not architectures:
            raise ValueError("No model architectures are specified")
1034
1035

        # Require transformers impl
1036
        if model_config.model_impl == "transformers":
1037
            arch = self._try_resolve_transformers(architectures[0], model_config)
1038
1039
1040
1041
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
1042
        elif model_config.model_impl == "terratorch":
1043
1044
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
1045

1046
        # Fallback to transformers impl (after resolving convert_type)
1047
1048
1049
1050
1051
1052
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1053
1054
1055
1056
1057
1058
1059
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1060
            model_info = self._try_inspect_model_cls(normalized_arch)
1061
            if model_info is not None:
1062
                return (model_info, arch)
1063

1064
        # Fallback to transformers impl (before resolving runner_type)
1065
1066
1067
1068
1069
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1070
1071
1072
1073
1074
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

1075
        return self._raise_for_unsupported(architectures)
1076

1077
1078
    def resolve_model_cls(
        self,
1079
        architectures: str | list[str],
1080
        model_config: ModelConfig,
1081
    ) -> tuple[type[nn.Module], str]:
1082
1083
        if isinstance(architectures, str):
            architectures = [architectures]
1084
1085
        if not architectures:
            raise ValueError("No model architectures are specified")
1086
1087

        # Require transformers impl
1088
        if model_config.model_impl == "transformers":
1089
            arch = self._try_resolve_transformers(architectures[0], model_config)
1090
1091
1092
1093
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1094
        elif model_config.model_impl == "terratorch":
1095
1096
1097
1098
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1099

1100
        # Fallback to transformers impl (after resolving convert_type)
1101
1102
1103
1104
1105
1106
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1107
1108
1109
1110
1111
1112
1113
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1114
            model_cls = self._try_load_model_cls(normalized_arch)
1115
1116
            if model_cls is not None:
                return (model_cls, arch)
1117

1118
        # Fallback to transformers impl (before resolving runner_type)
1119
1120
1121
1122
1123
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1124
1125
1126
1127
1128
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1129
        return self._raise_for_unsupported(architectures)
1130

1131
1132
    def is_text_generation_model(
        self,
1133
        architectures: str | list[str],
1134
        model_config: ModelConfig,
1135
    ) -> bool:
1136
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1137
        return model_cls.is_text_generation_model
1138

1139
    def is_pooling_model(
1140
        self,
1141
        architectures: str | list[str],
1142
        model_config: ModelConfig,
1143
    ) -> bool:
1144
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1145
        return model_cls.is_pooling_model
1146

1147
1148
    def is_cross_encoder_model(
        self,
1149
        architectures: str | list[str],
1150
        model_config: ModelConfig,
1151
    ) -> bool:
1152
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1153
        return model_cls.supports_cross_encoding
1154

1155
1156
    def is_multimodal_model(
        self,
1157
        architectures: str | list[str],
1158
        model_config: ModelConfig,
1159
    ) -> bool:
1160
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1161
        return model_cls.supports_multimodal
1162

1163
    def is_multimodal_raw_input_only_model(
1164
        self,
1165
        architectures: str | list[str],
1166
        model_config: ModelConfig,
1167
    ) -> bool:
1168
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1169
        return model_cls.supports_multimodal_raw_input_only
1170

1171
1172
    def is_pp_supported_model(
        self,
1173
        architectures: str | list[str],
1174
        model_config: ModelConfig,
1175
    ) -> bool:
1176
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1177
        return model_cls.supports_pp
1178

1179
1180
    def model_has_inner_state(
        self,
1181
        architectures: str | list[str],
1182
        model_config: ModelConfig,
1183
    ) -> bool:
1184
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1185
        return model_cls.has_inner_state
1186

1187
1188
    def is_attention_free_model(
        self,
1189
        architectures: str | list[str],
1190
        model_config: ModelConfig,
1191
    ) -> bool:
1192
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1193
        return model_cls.is_attention_free
1194

1195
1196
    def is_hybrid_model(
        self,
1197
        architectures: str | list[str],
1198
        model_config: ModelConfig,
1199
    ) -> bool:
1200
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1201
1202
        return model_cls.is_hybrid

1203
1204
    def is_noops_model(
        self,
1205
        architectures: str | list[str],
1206
        model_config: ModelConfig,
1207
    ) -> bool:
1208
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1209
1210
        return model_cls.has_noops

1211
1212
    def is_transcription_model(
        self,
1213
        architectures: str | list[str],
1214
        model_config: ModelConfig,
1215
    ) -> bool:
1216
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1217
1218
        return model_cls.supports_transcription

1219
1220
    def is_transcription_only_model(
        self,
1221
        architectures: str | list[str],
1222
        model_config: ModelConfig,
1223
    ) -> bool:
1224
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1225
1226
        return model_cls.supports_transcription_only

1227

1228
1229
1230
1231
1232
1233
1234
1235
1236
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1237
1238
1239
1240
1241

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1242
1243
1244
1245
1246
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1247
        # `cloudpickle` allows pickling lambda functions directly
1248
        import cloudpickle
1249

1250
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1251
1252
1253

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1254
1255
1256
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1257
1258
1259
1260
1261
1262

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1263
1264
1265
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1266

1267
        with open(output_filepath, "rb") as f:
1268
1269
1270
1271
1272
1273
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1274

1275
1276
1277
1278
1279
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1280
1281
1282

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1283
1284
1285


if __name__ == "__main__":
1286
    _run()