registry.py 52.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.tasks import ScoreType
34
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
35
from vllm.utils.hashing import safe_hash
36

37
38
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
39
    from vllm.config.pooler import SequencePoolingType, TokenPoolingType
40
41
else:
    AttnTypeStr = Any
42
43
    SequencePoolingType = Any
    TokenPoolingType = Any
44
45


46
47
48
49
50
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
51
    requires_raw_input_tokens,
52
    supports_mamba_prefix_caching,
53
54
55
56
57
58
59
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
60
    get_attn_type,
61
62
    get_default_seq_pooling_type,
    get_default_tok_pooling_type,
63
    get_score_type,
64
65
66
    is_pooling_model,
    is_text_generation_model,
)
67
68
69

logger = init_logger(__name__)

70
71
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
72
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
73
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
74
75
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
76
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
77
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
78
    "AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
79
80
81
82
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
83
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
84
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
85
    "BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
86
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
87
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
88
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
89
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
90
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
91
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
92
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
93
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
94
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
95
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
96
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
97
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
98
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
99
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
100
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
101
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
102
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
103
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
Kyungmin Lee's avatar
Kyungmin Lee committed
104
    "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
105
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
106
107
108
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
109
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
110
111
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
112
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
113
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
114
    "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
115
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
116
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
117
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
118
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
119
    "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
Jee Jee Li's avatar
Jee Jee Li committed
120
    "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
121
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
122
123
124
125
126
127
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
128
129
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
130
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
131
132
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
133
134
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
135
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
136
    "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
137
    "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
138
139
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
140
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
141
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
142
143
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
144
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
145
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
146
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
147
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
148
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
149
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
150
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
151
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
152
153
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
154
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
155
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
156
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
157
158
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
159
160
161
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
162
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
163
    "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
164
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
165
166
167
168
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
169
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
170
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
171
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
172
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
173
    "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
174
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
175
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
176
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
177
    "OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
178
179
180
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
181
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
182
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
183
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
184
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
185
186
187
188
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
189
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
190
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
191
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
192
193
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
194
195
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
196
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
197
198
    "SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
    "SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
199
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Li Xie's avatar
Li Xie committed
200
    "Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
Song's avatar
Song committed
201
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
csy0225's avatar
csy0225 committed
202
    "Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
203
204
205
206
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
207
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
208
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
209
    "TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
210
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
211
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
212
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
213
214
215
}

_EMBEDDING_MODELS = {
216
    # [Text-only]
217
    "BertModel": ("bert", "BertEmbeddingModel"),
218
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
219
    "ErnieModel": ("ernie", "ErnieEmbeddingModel"),
220
    "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
221
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
222
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
223
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
224
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
225
    "GritLM": ("gritlm", "GritLM"),
226
227
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
228
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
229
    "LlamaModel": ("llama", "LlamaForCausalLM"),
230
231
    **{
        # Multiple models share the same architecture, so we include them all
232
233
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
234
235
        if arch == "LlamaForCausalLM"
    },
236
    "MistralModel": ("llama", "LlamaForCausalLM"),
237
    "ModernBertModel": ("modernbert", "ModernBertModel"),
238
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
239
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
240
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
241
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
242
243
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
244
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
245
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
chengchengpei's avatar
chengchengpei committed
246
247
248
249
    "VoyageQwen3BidirectionalEmbedModel": (
        "voyage",
        "VoyageQwen3BidirectionalEmbedModel",
    ),
250
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
251
    # [Multimodal]
252
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
253
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
254
    "LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
255
256
257
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
258
    ),
259
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
260
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
261
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
262
263
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
264
    # models for the time being.
265
266
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
267
268
}

269
270
271
272
273
_LATE_INTERACTION_MODELS = {
    # [Text-only]
    "HF_ColBERT": ("colbert", "ColBERTModel"),
    "ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
    "ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
274
    "ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
275
276
    # [Multimodal]
    "ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
277
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
278
279
    "ColQwen3": ("colqwen3", "ColQwen3Model"),
    "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
280
    "ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
281
282
283
284
285
286
287
288
289
290
    "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
}

_REWARD_MODELS = {
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
}

_TOKEN_CLASSIFICATION_MODELS = {
291
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
292
    "ErnieForTokenClassification": ("ernie", "ErnieForTokenClassification"),
293
294
295
296
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
297
298
299
300
    "Qwen3ASRForcedAlignerForTokenClassification": (
        "qwen3_asr_forced_aligner",
        "Qwen3ASRForcedAlignerForTokenClassification",
    ),
301
302
303
304
305
}

_SEQUENCE_CLASSIFICATION_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
306
    "ErnieForSequenceClassification": ("ernie", "ErnieForSequenceClassification"),
307
308
309
310
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
311
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),
312
313
314
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
315
    ),
316
317
318
319
320
321
322
323
324
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
325
326
327
328
329
330
    # [Multimodal]
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaNemotronVLForSequenceClassification": (
        "nemotron_vl",
        "LlamaNemotronVLForSequenceClassification",
    ),
331
332
}

333
_MULTIMODAL_MODELS = {
334
    # [Decoder-only]
335
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
336
337
338
339
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
340
341
342
343
    "MusicFlamingoForConditionalGeneration": (
        "musicflamingo",
        "MusicFlamingoForConditionalGeneration",
    ),
344
345
346
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
347
    ),
348
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
349
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
350
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
351
352
353
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
354
    ),
355
356
    "Cheers": ("cheers", "CheersForConditionalGeneration"),
    "CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
357
358
359
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
360
    ),
361
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
362
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
RED's avatar
RED committed
363
    "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
364
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
365
366
367
368
    "Eagle2_5_VLForConditionalGeneration": (
        "eagle2_5_vl",
        "Eagle2_5_VLForConditionalGeneration",
    ),
369
370
371
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
372
    ),
373
374
375
376
    "FireRedASR2ForConditionalGeneration": (
        "fireredasr2",
        "FireRedASR2ForConditionalGeneration",
    ),
377
    "FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
378
379
380
381
    "FunAudioChatForConditionalGeneration": (
        "funaudiochat",
        "FunAudioChatForConditionalGeneration",
    ),
382
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
383
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),
384
385
386
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
387
    ),
388
    "Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
389
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
390
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
391
392
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
393
    "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
394
395
396
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
397
    ),
398
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
399
400
401
402
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
403
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
404
405
406
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
407
    ),
408
409
410
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
411
    ),
zxy's avatar
zxy committed
412
413
414
415
    "InternS1ProForConditionalGeneration": (
        "interns1_pro",
        "InternS1ProForConditionalGeneration",
    ),
416
417
418
419
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
420
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
421
    "KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
422
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
423
424
425
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
426
    ),
427
428
429
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
    "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
    "MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
430
431
432
433
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
434
    "Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
435
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),
436
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
437
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
438
439
440
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
441
    ),
442
443
444
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
445
    ),
446
447
448
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
449
    ),
450
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
451
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
452
453
454
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
455
    ),
456
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
457
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
458
459
460
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
461
    ),
462
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
463
    "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
464
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
465
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
466
    "OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
467
468
469
470
    "OpenPanguVLForConditionalGeneration": (
        "openpangu_vl",
        "OpenPanguVLForConditionalGeneration",
    ),
471
    "Ovis": ("ovis", "Ovis"),
472
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
473
474
    "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
    "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
475
476
477
478
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
479
480
481
482
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
483
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
484
    "Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
485
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
486
487
488
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
489
490
491
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
492
    ),
493
494
495
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
496
    ),
497
498
499
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
500
    ),
501
502
503
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
504
    ),
505
506
507
508
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
Roger Wang's avatar
Roger Wang committed
509
510
511
512
    "Qwen3ASRForConditionalGeneration": (
        "qwen3_asr",
        "Qwen3ASRForConditionalGeneration",
    ),
513
514
    "Qwen3ASRRealtimeGeneration": ("qwen3_asr_realtime", "Qwen3ASRRealtimeGeneration"),
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
515
516
517
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
518
    ),
519
    "Qwen3_5ForConditionalGeneration": ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
520
521
522
523
    "Qwen3_5MoeForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5MoeForConditionalGeneration",
    ),
524
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
525
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
526
527
528
529
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
    "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
530
531
532
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
533
    ),
534
    "UltravoxModel": ("ultravox", "UltravoxModel"),
535
536
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
    "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
537
    # [Encoder-decoder]
538
    "CohereAsrForConditionalGeneration": (
Ekagra Ranjan's avatar
Ekagra Ranjan committed
539
        "cohere_asr",
540
        "CohereAsrForConditionalGeneration",
Ekagra Ranjan's avatar
Ekagra Ranjan committed
541
    ),
542
543
544
545
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
546
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),
547
}
548
549

_SPECULATIVE_DECODING_MODELS = {
550
    "ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
551
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
552
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
553
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
554
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
555
    "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
556
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
557
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
558
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
559
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
560
561
562
563
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
564
565
    "Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
    "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
566
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
567
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
568
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
569
    "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
570
    "NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
XuruiYang's avatar
XuruiYang committed
571
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
572
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
573
    "Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
574
    "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
575
    "MedusaModel": ("medusa", "Medusa"),
576
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
577
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
csy0225's avatar
csy0225 committed
578
    "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
579
580
    "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
    "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
581
582
583
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
584
}
585

586
_TRANSFORMERS_SUPPORTED_MODELS = {
587
588
589
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
590
591
592
593
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
594
595
596
}

_TRANSFORMERS_BACKEND_MODELS = {
597
    # Text generation models
598
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
617
    "TransformersForSequenceClassification": (
618
        "transformers",
619
        "TransformersForSequenceClassification",
620
    ),
621
    "TransformersMoEForSequenceClassification": (
622
        "transformers",
623
        "TransformersMoEForSequenceClassification",
624
    ),
625
626
627
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
628
    ),
629
}
630

631
_VLLM_MODELS = {
632
    **_TEXT_GENERATION_MODELS,
633
    **_EMBEDDING_MODELS,
634
635
636
637
    **_LATE_INTERACTION_MODELS,
    **_REWARD_MODELS,
    **_TOKEN_CLASSIFICATION_MODELS,
    **_SEQUENCE_CLASSIFICATION_MODELS,
638
    **_MULTIMODAL_MODELS,
639
    **_SPECULATIVE_DECODING_MODELS,
640
641
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
642
643
}

644
645
646
647
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
648
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
649

650
_PREVIOUSLY_SUPPORTED_MODELS = {
651
    "MotifForCausalLM": "0.10.2",
652
    "Phi3SmallForCausalLM": "0.9.2",
653
    "Phi4FlashForCausalLM": "0.10.2",
654
    "Phi4MultimodalForCausalLM": "0.12.0",
655
656
657
658
659
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "DonutForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
660

661
662
663
664
665
666
667
_OOT_SUPPORTED_MODELS = {
    "BartModel": "https://github.com/vllm-project/bart-plugin",
    "BartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "Florence2ForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "MBartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
}

668

669
670
@dataclass(frozen=True)
class _ModelInfo:
671
    architecture: str
672
    is_text_generation_model: bool
673
    is_pooling_model: bool
674
    attn_type: AttnTypeStr
675
676
    default_seq_pooling_type: SequencePoolingType
    default_tok_pooling_type: TokenPoolingType
677
    score_type: ScoreType
678
    supports_multimodal: bool
679
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
680
    requires_raw_input_tokens: bool
681
    supports_multimodal_encoder_tp_data: bool
682
    supports_pp: bool
683
684
    has_inner_state: bool
    is_attention_free: bool
685
    is_hybrid: bool
686
    has_noops: bool
687
    supports_mamba_prefix_caching: bool
688
    supports_transcription: bool
689
    supports_transcription_only: bool
690
691

    @staticmethod
692
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
693
        return _ModelInfo(
694
            architecture=model.__name__,
695
            is_text_generation_model=is_text_generation_model(model),
696
            is_pooling_model=is_pooling_model(model),
697
698
            default_seq_pooling_type=get_default_seq_pooling_type(model),
            default_tok_pooling_type=get_default_tok_pooling_type(model),
699
            attn_type=get_attn_type(model),
700
            score_type=get_score_type(model),
701
            supports_multimodal=supports_multimodal(model),
702
703
704
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
705
            requires_raw_input_tokens=requires_raw_input_tokens(model),
706
707
708
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
709
            supports_pp=supports_pp(model),
710
711
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
712
            is_hybrid=is_hybrid(model),
713
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
714
            supports_transcription=supports_transcription(model),
715
716
717
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
718
            has_noops=has_noops(model),
719
        )
720
721


722
723
724
725
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
726

727
    @abstractmethod
728
    def load_model_cls(self) -> type[nn.Module]:
729
        raise NotImplementedError
730
731


732
733
734
735
736
737
738
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
739
    model_cls: type[nn.Module]
740
741

    @staticmethod
742
    def from_model_cls(model_cls: type[nn.Module]):
743
744
745
746
747
748
749
750
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

751
    def load_model_cls(self) -> type[nn.Module]:
752
753
754
755
756
757
758
759
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
760

761
762
763
    module_name: str
    class_name: str

764
765
766
767
768
769
770
771
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

772
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
773
774
        try:
            try:
775
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
776
777
778
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
779
                logger.debug(
780
                    "Cached model info file for class %s.%s not found",
781
782
783
                    self.module_name,
                    self.class_name,
                )
784
785
786
                return None

            if mi_dict["hash"] != module_hash:
787
                logger.debug(
788
                    "Cached model info file for class %s.%s is stale",
789
790
791
                    self.module_name,
                    self.class_name,
                )
792
793
794
795
796
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
797
            logger.debug(
798
                "Cached model info for class %s.%s error. ",
799
800
801
                self.module_name,
                self.class_name,
            )
802
803
            return None

804
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
805
806
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
807

808
809
810
811
812
813
814
815
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
816
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
817
818
819
820
821
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
822
    def inspect_model_cls(self) -> _ModelInfo:
823
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
824
        module_hash = None
825

826
827
        if model_path.exists():
            with open(model_path, "rb") as f:
828
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
829
830
831

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
832
                logger.debug(
833
                    "Loaded model info for class %s.%s from cache",
834
835
836
                    self.module_name,
                    self.class_name,
                )
837
838
                return mi
            else:
839
                logger.debug(
840
                    "Cache model info for class %s.%s miss. Loading model instead.",
841
842
843
                    self.module_name,
                    self.class_name,
                )
844
845
846

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
847
848
849
850
851
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
852
853

        # save cache file
854
855
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
856
857

        return mi
858

859
    def load_model_cls(self) -> type[nn.Module]:
860
861
862
863
864
865
866
867
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
868
) -> type[nn.Module] | None:
869
    from vllm.platforms import current_platform
870

871
    current_platform.verify_model_arch(model_arch)
872
873
874
    try:
        return model.load_model_cls()
    except Exception:
875
        logger.exception("Error in loading model architecture '%s'", model_arch)
876
        return None
877
878


879
880
881
882
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
883
) -> _ModelInfo | None:
884
885
886
    try:
        return model.inspect_model_cls()
    except Exception:
887
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
888
        return None
889
890


891
892
893
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
894
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
895

896
    def get_supported_archs(self) -> Set[str]:
897
        return self.models.keys()
898

899
900
901
    def register_model(
        self,
        model_arch: str,
902
        model_cls: type[nn.Module] | str,
903
    ) -> None:
904
905
906
        """
        Register an external model to be used in vLLM.

907
        `model_cls` can be either:
908

909
        - A [`torch.nn.Module`][] class directly referencing the model.
910
        - A string in the format `<module>:<class>` which can be used to
911
912
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
913
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
914
        """
915
916
917
918
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

919
        if model_arch in self.models:
920
921
            logger.warning(
                "Model architecture %s is already registered, and will be "
922
923
924
925
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
926
927
928
929
930
931

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
932

933
            model = _LazyRegisteredModel(*split_str)
934
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
935
            model = _RegisteredModel.from_model_cls(model_cls)
936
        else:
937
938
939
940
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
941
            raise TypeError(msg)
942

943
        self.models[model_arch] = model
944

945
    def _raise_for_unsupported(self, architectures: list[str]):
946
        all_supported_archs = self.get_supported_archs()
947

948
949
950
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
951
952
                "to be inspected. Please check the logs for more details."
            )
953

954
955
956
957
958
959
960
961
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
962
963
                    "use this model architecture."
                )
964
965
966
967
968
969
970
971
            if arch in _OOT_SUPPORTED_MODELS:
                plugin_url = _OOT_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} is not supported in-tree anymore. "
                    f"Please install the plugin at {plugin_url} if you want to "
                    "use this model architecture."
                )
972

973
974
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
975
976
            f"Supported architectures: {all_supported_archs}"
        )
977

978
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
979
980
        if model_arch not in self.models:
            return None
981

982
        return _try_load_model_cls(model_arch, self.models[model_arch])
983

984
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
985
986
        if model_arch not in self.models:
            return None
987

988
989
990
991
992
993
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
994
    ) -> str | None:
995
996
997
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

998
999
1000
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1017
                        trust_remote_code=model_config.trust_remote_code,
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1030
                        trust_remote_code=model_config.trust_remote_code,
1031
1032
1033
1034
1035
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
1036
                if model_config.model_impl != "transformers":
1037
1038
1039
1040
1041
1042
1043
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
1044
1045
                    "'auto_map' (relevant if the model is custom)."
                )
1046
1047

        if not model_module.is_backend_compatible():
1048
            if model_config.model_impl != "transformers":
1049
                return None
1050

1051
1052
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
1053
1054
                "is not compatible with vLLM."
            )
1055

1056
        return model_config._get_transformers_backend_cls()
1057

1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
1083

1084
1085
    def inspect_model_cls(
        self,
1086
        architectures: str | list[str],
1087
        model_config: ModelConfig,
1088
    ) -> tuple[_ModelInfo, str]:
1089
1090
        if isinstance(architectures, str):
            architectures = [architectures]
1091
1092
        if not architectures:
            raise ValueError("No model architectures are specified")
1093
1094

        # Require transformers impl
1095
        if model_config.model_impl == "transformers":
1096
            arch = self._try_resolve_transformers(architectures[0], model_config)
1097
1098
1099
1100
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
1101
        elif model_config.model_impl == "terratorch":
1102
1103
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
1104

1105
        # Fallback to transformers impl (after resolving convert_type)
1106
1107
1108
1109
1110
1111
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1112
1113
1114
1115
1116
1117
1118
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1119
            model_info = self._try_inspect_model_cls(normalized_arch)
1120
            if model_info is not None:
1121
                return (model_info, arch)
1122

1123
        # Fallback to transformers impl (before resolving runner_type)
1124
1125
1126
1127
1128
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1129
1130
1131
1132
1133
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

1134
        return self._raise_for_unsupported(architectures)
1135

1136
1137
    def resolve_model_cls(
        self,
1138
        architectures: str | list[str],
1139
        model_config: ModelConfig,
1140
    ) -> tuple[type[nn.Module], str]:
1141
1142
        if isinstance(architectures, str):
            architectures = [architectures]
1143
1144
        if not architectures:
            raise ValueError("No model architectures are specified")
1145
1146

        # Require transformers impl
1147
        if model_config.model_impl == "transformers":
1148
            arch = self._try_resolve_transformers(architectures[0], model_config)
1149
1150
1151
1152
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1153
        elif model_config.model_impl == "terratorch":
1154
1155
1156
1157
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1158

1159
        # Fallback to transformers impl (after resolving convert_type)
1160
1161
1162
1163
1164
1165
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1166
1167
1168
1169
1170
1171
1172
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1173
            model_cls = self._try_load_model_cls(normalized_arch)
1174
1175
            if model_cls is not None:
                return (model_cls, arch)
1176

1177
        # Fallback to transformers impl (before resolving runner_type)
1178
1179
1180
1181
1182
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1183
1184
1185
1186
1187
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1188
        return self._raise_for_unsupported(architectures)
1189

1190
1191
    def is_text_generation_model(
        self,
1192
        architectures: str | list[str],
1193
        model_config: ModelConfig,
1194
    ) -> bool:
1195
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1196
        return model_cls.is_text_generation_model
1197

1198
    def is_pooling_model(
1199
        self,
1200
        architectures: str | list[str],
1201
        model_config: ModelConfig,
1202
    ) -> bool:
1203
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1204
        return model_cls.is_pooling_model
1205
1206
1207

    def is_multimodal_model(
        self,
1208
        architectures: str | list[str],
1209
        model_config: ModelConfig,
1210
    ) -> bool:
1211
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1212
        return model_cls.supports_multimodal
1213

1214
    def is_multimodal_raw_input_only_model(
1215
        self,
1216
        architectures: str | list[str],
1217
        model_config: ModelConfig,
1218
    ) -> bool:
1219
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1220
        return model_cls.supports_multimodal_raw_input_only
1221

1222
1223
    def is_pp_supported_model(
        self,
1224
        architectures: str | list[str],
1225
        model_config: ModelConfig,
1226
    ) -> bool:
1227
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1228
        return model_cls.supports_pp
1229

1230
1231
    def model_has_inner_state(
        self,
1232
        architectures: str | list[str],
1233
        model_config: ModelConfig,
1234
    ) -> bool:
1235
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1236
        return model_cls.has_inner_state
1237

1238
1239
    def is_attention_free_model(
        self,
1240
        architectures: str | list[str],
1241
        model_config: ModelConfig,
1242
    ) -> bool:
1243
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1244
        return model_cls.is_attention_free
1245

1246
1247
    def is_hybrid_model(
        self,
1248
        architectures: str | list[str],
1249
        model_config: ModelConfig,
1250
    ) -> bool:
1251
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1252
1253
        return model_cls.is_hybrid

1254
1255
    def is_noops_model(
        self,
1256
        architectures: str | list[str],
1257
        model_config: ModelConfig,
1258
    ) -> bool:
1259
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1260
1261
        return model_cls.has_noops

1262
1263
    def is_transcription_model(
        self,
1264
        architectures: str | list[str],
1265
        model_config: ModelConfig,
1266
    ) -> bool:
1267
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1268
1269
        return model_cls.supports_transcription

1270
1271
    def is_transcription_only_model(
        self,
1272
        architectures: str | list[str],
1273
        model_config: ModelConfig,
1274
    ) -> bool:
1275
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1276
1277
        return model_cls.supports_transcription_only

1278

1279
1280
1281
1282
1283
1284
1285
1286
1287
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1288
1289
1290
1291
1292

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1293
1294
1295
1296
1297
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1298
        # `cloudpickle` allows pickling lambda functions directly
1299
        import cloudpickle
1300

1301
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1302
1303
1304

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1305
1306
1307
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1308
1309
1310
1311
1312
1313

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1314
1315
1316
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1317

1318
        with open(output_filepath, "rb") as f:
1319
1320
1321
1322
1323
1324
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1325

1326
1327
1328
1329
1330
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1331
1332
1333

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1334
1335
1336


if __name__ == "__main__":
1337
    _run()