registry.py 52.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.tasks import ScoreType
34
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
35
from vllm.utils.hashing import safe_hash
36

37
38
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
39
    from vllm.config.pooler import SequencePoolingType, TokenPoolingType
40
41
else:
    AttnTypeStr = Any
42
43
    SequencePoolingType = Any
    TokenPoolingType = Any
44
45


46
47
48
49
50
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
51
    requires_raw_input_tokens,
52
    supports_mamba_prefix_caching,
53
54
55
56
57
58
59
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
60
    get_attn_type,
61
62
    get_default_seq_pooling_type,
    get_default_tok_pooling_type,
63
    get_score_type,
64
65
66
    is_pooling_model,
    is_text_generation_model,
)
67
68
69

logger = init_logger(__name__)

70
71
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
72
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
73
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
74
75
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
76
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
77
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
78
    "AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
79
80
81
82
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
83
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
84
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
85
    "BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
86
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
87
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
88
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
89
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
90
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
91
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
92
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
93
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
94
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
95
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
96
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
97
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
98
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
99
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
100
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
101
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
102
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
103
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
Kyungmin Lee's avatar
Kyungmin Lee committed
104
    "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
105
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
106
107
108
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
109
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
110
111
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
112
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
113
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
114
    "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
115
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
116
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
117
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
118
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
119
    "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
Jee Jee Li's avatar
Jee Jee Li committed
120
    "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
121
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
122
123
124
125
126
127
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
128
129
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
130
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
131
132
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
133
134
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
135
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
136
    "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
137
    "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
138
139
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
140
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
141
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
142
143
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
144
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
145
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
146
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
147
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
148
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
149
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
150
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
151
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
152
153
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
154
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
155
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
156
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
157
158
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
159
160
161
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
162
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
163
    "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
164
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
165
166
167
168
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
169
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
170
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
171
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
172
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
173
    "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
174
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
175
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
176
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
177
    "OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
178
179
180
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
181
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
182
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
183
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
184
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
185
186
187
188
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
189
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
190
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
191
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
192
193
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
194
195
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
196
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
197
198
    "SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
    "SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
199
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Li Xie's avatar
Li Xie committed
200
    "Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
Song's avatar
Song committed
201
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
csy0225's avatar
csy0225 committed
202
    "Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
203
204
205
206
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
207
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
208
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
209
    "TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
210
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
211
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
212
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
213
214
215
}

_EMBEDDING_MODELS = {
216
    # [Text-only]
217
    "BertModel": ("bert", "BertEmbeddingModel"),
218
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
219
    "ErnieModel": ("ernie", "ErnieEmbeddingModel"),
220
    "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
221
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
222
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
223
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
224
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
225
    "GritLM": ("gritlm", "GritLM"),
226
227
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
228
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
229
    "LlamaModel": ("llama", "LlamaForCausalLM"),
230
231
    **{
        # Multiple models share the same architecture, so we include them all
232
233
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
234
235
        if arch == "LlamaForCausalLM"
    },
236
    "MistralModel": ("llama", "LlamaForCausalLM"),
237
    "ModernBertModel": ("modernbert", "ModernBertModel"),
238
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
239
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
240
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
241
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
242
243
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
244
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
245
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
chengchengpei's avatar
chengchengpei committed
246
247
248
249
    "VoyageQwen3BidirectionalEmbedModel": (
        "voyage",
        "VoyageQwen3BidirectionalEmbedModel",
    ),
250
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
251
    # [Multimodal]
252
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
253
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
254
    "LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
255
256
257
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
258
    ),
259
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
260
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
261
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
262
263
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
264
    # models for the time being.
265
266
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
267
268
}

269
270
271
272
273
_LATE_INTERACTION_MODELS = {
    # [Text-only]
    "HF_ColBERT": ("colbert", "ColBERTModel"),
    "ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
    "ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
274
    "ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
275
276
    # [Multimodal]
    "ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
277
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
278
279
    "ColQwen3": ("colqwen3", "ColQwen3Model"),
    "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
280
    "ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
281
282
283
284
285
286
287
288
289
290
    "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
}

_REWARD_MODELS = {
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
}

_TOKEN_CLASSIFICATION_MODELS = {
291
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
292
    "ErnieForTokenClassification": ("ernie", "ErnieForTokenClassification"),
293
294
295
296
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
297
298
299
300
    "Qwen3ASRForcedAlignerForTokenClassification": (
        "qwen3_asr_forced_aligner",
        "Qwen3ASRForcedAlignerForTokenClassification",
    ),
301
302
303
304
305
}

_SEQUENCE_CLASSIFICATION_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
306
    "ErnieForSequenceClassification": ("ernie", "ErnieForSequenceClassification"),
307
308
309
310
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
311
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),
312
313
314
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
315
    ),
316
317
318
319
320
321
322
323
324
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
325
326
327
328
329
330
    # [Multimodal]
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaNemotronVLForSequenceClassification": (
        "nemotron_vl",
        "LlamaNemotronVLForSequenceClassification",
    ),
331
332
}

333
_MULTIMODAL_MODELS = {
334
    # [Decoder-only]
335
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
336
337
338
339
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
340
341
342
343
    "MusicFlamingoForConditionalGeneration": (
        "musicflamingo",
        "MusicFlamingoForConditionalGeneration",
    ),
344
345
346
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
347
    ),
348
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
349
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
350
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
351
352
353
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
354
    ),
355
356
    "Cheers": ("cheers", "CheersForConditionalGeneration"),
    "CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
357
358
359
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
360
    ),
361
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
362
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
RED's avatar
RED committed
363
    "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
364
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
365
366
367
368
    "Eagle2_5_VLForConditionalGeneration": (
        "eagle2_5_vl",
        "Eagle2_5_VLForConditionalGeneration",
    ),
369
370
371
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
372
    ),
373
374
375
376
    "FireRedASR2ForConditionalGeneration": (
        "fireredasr2",
        "FireRedASR2ForConditionalGeneration",
    ),
377
    "FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
378
379
380
381
    "FunAudioChatForConditionalGeneration": (
        "funaudiochat",
        "FunAudioChatForConditionalGeneration",
    ),
382
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
383
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),
384
385
386
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
387
    ),
388
    "Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
389
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
390
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
391
392
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
393
    "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
394
395
396
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
397
    ),
398
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
399
400
401
402
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
403
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
404
405
406
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
407
    ),
408
409
410
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
411
    ),
zxy's avatar
zxy committed
412
413
414
415
    "InternS1ProForConditionalGeneration": (
        "interns1_pro",
        "InternS1ProForConditionalGeneration",
    ),
416
417
418
419
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
420
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
421
    "KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
422
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
423
424
425
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
426
    ),
427
428
429
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
    "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
    "MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
430
431
432
433
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
434
    "Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
435
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),
436
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
437
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
438
439
440
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
441
    ),
442
443
444
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
445
    ),
446
447
448
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
449
    ),
450
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
451
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
452
453
454
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
455
    ),
456
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
457
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
458
459
460
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
461
    ),
462
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
463
    "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
464
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
465
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
466
    "OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
467
468
469
470
    "OpenPanguVLForConditionalGeneration": (
        "openpangu_vl",
        "OpenPanguVLForConditionalGeneration",
    ),
471
    "Ovis": ("ovis", "Ovis"),
472
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
473
474
    "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
    "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
475
476
477
478
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
479
480
481
482
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
483
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
484
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
485
486
487
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
488
489
490
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
491
    ),
492
493
494
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
495
    ),
496
497
498
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
499
    ),
500
501
502
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
503
    ),
504
505
506
507
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
Roger Wang's avatar
Roger Wang committed
508
509
510
511
    "Qwen3ASRForConditionalGeneration": (
        "qwen3_asr",
        "Qwen3ASRForConditionalGeneration",
    ),
512
513
    "Qwen3ASRRealtimeGeneration": ("qwen3_asr_realtime", "Qwen3ASRRealtimeGeneration"),
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
514
515
516
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
517
    ),
518
    "Qwen3_5ForConditionalGeneration": ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
519
520
521
522
    "Qwen3_5MoeForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5MoeForConditionalGeneration",
    ),
523
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
524
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
525
526
527
528
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
    "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
529
530
531
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
532
    ),
533
    "UltravoxModel": ("ultravox", "UltravoxModel"),
534
535
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
    "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
536
    # [Encoder-decoder]
537
    "CohereAsrForConditionalGeneration": (
Ekagra Ranjan's avatar
Ekagra Ranjan committed
538
        "cohere_asr",
539
        "CohereAsrForConditionalGeneration",
Ekagra Ranjan's avatar
Ekagra Ranjan committed
540
    ),
541
542
543
544
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
545
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),
546
}
547
548

_SPECULATIVE_DECODING_MODELS = {
549
    "ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
550
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
551
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
552
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
553
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
554
    "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
555
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
556
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
557
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
558
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
559
560
561
562
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
563
564
    "Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
    "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
565
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
566
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
567
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
568
    "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
569
    "NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
XuruiYang's avatar
XuruiYang committed
570
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
571
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
572
    "Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
573
    "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
574
    "MedusaModel": ("medusa", "Medusa"),
575
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
576
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
csy0225's avatar
csy0225 committed
577
    "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
578
579
    "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
    "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
580
581
582
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
583
}
584

585
_TRANSFORMERS_SUPPORTED_MODELS = {
586
587
588
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
589
590
591
592
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
593
594
595
}

_TRANSFORMERS_BACKEND_MODELS = {
596
    # Text generation models
597
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
616
    "TransformersForSequenceClassification": (
617
        "transformers",
618
        "TransformersForSequenceClassification",
619
    ),
620
    "TransformersMoEForSequenceClassification": (
621
        "transformers",
622
        "TransformersMoEForSequenceClassification",
623
    ),
624
625
626
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
627
    ),
628
}
629

630
_VLLM_MODELS = {
631
    **_TEXT_GENERATION_MODELS,
632
    **_EMBEDDING_MODELS,
633
634
635
636
    **_LATE_INTERACTION_MODELS,
    **_REWARD_MODELS,
    **_TOKEN_CLASSIFICATION_MODELS,
    **_SEQUENCE_CLASSIFICATION_MODELS,
637
    **_MULTIMODAL_MODELS,
638
    **_SPECULATIVE_DECODING_MODELS,
639
640
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
641
642
}

643
644
645
646
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
647
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
648

649
_PREVIOUSLY_SUPPORTED_MODELS = {
650
    "MotifForCausalLM": "0.10.2",
651
    "Phi3SmallForCausalLM": "0.9.2",
652
    "Phi4FlashForCausalLM": "0.10.2",
653
    "Phi4MultimodalForCausalLM": "0.12.0",
654
655
656
657
658
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "DonutForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
659

660
661
662
663
664
665
666
_OOT_SUPPORTED_MODELS = {
    "BartModel": "https://github.com/vllm-project/bart-plugin",
    "BartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "Florence2ForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "MBartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
}

667

668
669
@dataclass(frozen=True)
class _ModelInfo:
670
    architecture: str
671
    is_text_generation_model: bool
672
    is_pooling_model: bool
673
    attn_type: AttnTypeStr
674
675
    default_seq_pooling_type: SequencePoolingType
    default_tok_pooling_type: TokenPoolingType
676
    score_type: ScoreType
677
    supports_multimodal: bool
678
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
679
    requires_raw_input_tokens: bool
680
    supports_multimodal_encoder_tp_data: bool
681
    supports_pp: bool
682
683
    has_inner_state: bool
    is_attention_free: bool
684
    is_hybrid: bool
685
    has_noops: bool
686
    supports_mamba_prefix_caching: bool
687
    supports_transcription: bool
688
    supports_transcription_only: bool
689
690

    @staticmethod
691
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
692
        return _ModelInfo(
693
            architecture=model.__name__,
694
            is_text_generation_model=is_text_generation_model(model),
695
            is_pooling_model=is_pooling_model(model),
696
697
            default_seq_pooling_type=get_default_seq_pooling_type(model),
            default_tok_pooling_type=get_default_tok_pooling_type(model),
698
            attn_type=get_attn_type(model),
699
            score_type=get_score_type(model),
700
            supports_multimodal=supports_multimodal(model),
701
702
703
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
704
            requires_raw_input_tokens=requires_raw_input_tokens(model),
705
706
707
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
708
            supports_pp=supports_pp(model),
709
710
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
711
            is_hybrid=is_hybrid(model),
712
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
713
            supports_transcription=supports_transcription(model),
714
715
716
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
717
            has_noops=has_noops(model),
718
        )
719
720


721
722
723
724
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
725

726
    @abstractmethod
727
    def load_model_cls(self) -> type[nn.Module]:
728
        raise NotImplementedError
729
730


731
732
733
734
735
736
737
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
738
    model_cls: type[nn.Module]
739
740

    @staticmethod
741
    def from_model_cls(model_cls: type[nn.Module]):
742
743
744
745
746
747
748
749
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

750
    def load_model_cls(self) -> type[nn.Module]:
751
752
753
754
755
756
757
758
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
759

760
761
762
    module_name: str
    class_name: str

763
764
765
766
767
768
769
770
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

771
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
772
773
        try:
            try:
774
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
775
776
777
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
778
                logger.debug(
779
                    "Cached model info file for class %s.%s not found",
780
781
782
                    self.module_name,
                    self.class_name,
                )
783
784
785
                return None

            if mi_dict["hash"] != module_hash:
786
                logger.debug(
787
                    "Cached model info file for class %s.%s is stale",
788
789
790
                    self.module_name,
                    self.class_name,
                )
791
792
793
794
795
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
796
            logger.debug(
797
                "Cached model info for class %s.%s error. ",
798
799
800
                self.module_name,
                self.class_name,
            )
801
802
            return None

803
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
804
805
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
806

807
808
809
810
811
812
813
814
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
815
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
816
817
818
819
820
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
821
    def inspect_model_cls(self) -> _ModelInfo:
822
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
823
        module_hash = None
824

825
826
        if model_path.exists():
            with open(model_path, "rb") as f:
827
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
828
829
830

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
831
                logger.debug(
832
                    "Loaded model info for class %s.%s from cache",
833
834
835
                    self.module_name,
                    self.class_name,
                )
836
837
                return mi
            else:
838
                logger.debug(
839
                    "Cache model info for class %s.%s miss. Loading model instead.",
840
841
842
                    self.module_name,
                    self.class_name,
                )
843
844
845

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
846
847
848
849
850
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
851
852

        # save cache file
853
854
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
855
856

        return mi
857

858
    def load_model_cls(self) -> type[nn.Module]:
859
860
861
862
863
864
865
866
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
867
) -> type[nn.Module] | None:
868
    from vllm.platforms import current_platform
869

870
    current_platform.verify_model_arch(model_arch)
871
872
873
    try:
        return model.load_model_cls()
    except Exception:
874
        logger.exception("Error in loading model architecture '%s'", model_arch)
875
        return None
876
877


878
879
880
881
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
882
) -> _ModelInfo | None:
883
884
885
    try:
        return model.inspect_model_cls()
    except Exception:
886
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
887
        return None
888
889


890
891
892
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
893
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
894

895
    def get_supported_archs(self) -> Set[str]:
896
        return self.models.keys()
897

898
899
900
    def register_model(
        self,
        model_arch: str,
901
        model_cls: type[nn.Module] | str,
902
    ) -> None:
903
904
905
        """
        Register an external model to be used in vLLM.

906
        `model_cls` can be either:
907

908
        - A [`torch.nn.Module`][] class directly referencing the model.
909
        - A string in the format `<module>:<class>` which can be used to
910
911
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
912
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
913
        """
914
915
916
917
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

918
        if model_arch in self.models:
919
920
            logger.warning(
                "Model architecture %s is already registered, and will be "
921
922
923
924
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
925
926
927
928
929
930

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
931

932
            model = _LazyRegisteredModel(*split_str)
933
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
934
            model = _RegisteredModel.from_model_cls(model_cls)
935
        else:
936
937
938
939
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
940
            raise TypeError(msg)
941

942
        self.models[model_arch] = model
943

944
    def _raise_for_unsupported(self, architectures: list[str]):
945
        all_supported_archs = self.get_supported_archs()
946

947
948
949
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
950
951
                "to be inspected. Please check the logs for more details."
            )
952

953
954
955
956
957
958
959
960
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
961
962
                    "use this model architecture."
                )
963
964
965
966
967
968
969
970
            if arch in _OOT_SUPPORTED_MODELS:
                plugin_url = _OOT_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} is not supported in-tree anymore. "
                    f"Please install the plugin at {plugin_url} if you want to "
                    "use this model architecture."
                )
971

972
973
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
974
975
            f"Supported architectures: {all_supported_archs}"
        )
976

977
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
978
979
        if model_arch not in self.models:
            return None
980

981
        return _try_load_model_cls(model_arch, self.models[model_arch])
982

983
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
984
985
        if model_arch not in self.models:
            return None
986

987
988
989
990
991
992
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
993
    ) -> str | None:
994
995
996
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

997
998
999
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1016
                        trust_remote_code=model_config.trust_remote_code,
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1029
                        trust_remote_code=model_config.trust_remote_code,
1030
1031
1032
1033
1034
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
1035
                if model_config.model_impl != "transformers":
1036
1037
1038
1039
1040
1041
1042
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
1043
1044
                    "'auto_map' (relevant if the model is custom)."
                )
1045
1046

        if not model_module.is_backend_compatible():
1047
            if model_config.model_impl != "transformers":
1048
                return None
1049

1050
1051
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
1052
1053
                "is not compatible with vLLM."
            )
1054

1055
        return model_config._get_transformers_backend_cls()
1056

1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
1082

1083
1084
    def inspect_model_cls(
        self,
1085
        architectures: str | list[str],
1086
        model_config: ModelConfig,
1087
    ) -> tuple[_ModelInfo, str]:
1088
1089
        if isinstance(architectures, str):
            architectures = [architectures]
1090
1091
        if not architectures:
            raise ValueError("No model architectures are specified")
1092
1093

        # Require transformers impl
1094
        if model_config.model_impl == "transformers":
1095
            arch = self._try_resolve_transformers(architectures[0], model_config)
1096
1097
1098
1099
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
1100
        elif model_config.model_impl == "terratorch":
1101
1102
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
1103

1104
        # Fallback to transformers impl (after resolving convert_type)
1105
1106
1107
1108
1109
1110
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1111
1112
1113
1114
1115
1116
1117
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1118
            model_info = self._try_inspect_model_cls(normalized_arch)
1119
            if model_info is not None:
1120
                return (model_info, arch)
1121

1122
        # Fallback to transformers impl (before resolving runner_type)
1123
1124
1125
1126
1127
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1128
1129
1130
1131
1132
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

1133
        return self._raise_for_unsupported(architectures)
1134

1135
1136
    def resolve_model_cls(
        self,
1137
        architectures: str | list[str],
1138
        model_config: ModelConfig,
1139
    ) -> tuple[type[nn.Module], str]:
1140
1141
        if isinstance(architectures, str):
            architectures = [architectures]
1142
1143
        if not architectures:
            raise ValueError("No model architectures are specified")
1144
1145

        # Require transformers impl
1146
        if model_config.model_impl == "transformers":
1147
            arch = self._try_resolve_transformers(architectures[0], model_config)
1148
1149
1150
1151
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1152
        elif model_config.model_impl == "terratorch":
1153
1154
1155
1156
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1157

1158
        # Fallback to transformers impl (after resolving convert_type)
1159
1160
1161
1162
1163
1164
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1165
1166
1167
1168
1169
1170
1171
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1172
            model_cls = self._try_load_model_cls(normalized_arch)
1173
1174
            if model_cls is not None:
                return (model_cls, arch)
1175

1176
        # Fallback to transformers impl (before resolving runner_type)
1177
1178
1179
1180
1181
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1182
1183
1184
1185
1186
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1187
        return self._raise_for_unsupported(architectures)
1188

1189
1190
    def is_text_generation_model(
        self,
1191
        architectures: str | list[str],
1192
        model_config: ModelConfig,
1193
    ) -> bool:
1194
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1195
        return model_cls.is_text_generation_model
1196

1197
    def is_pooling_model(
1198
        self,
1199
        architectures: str | list[str],
1200
        model_config: ModelConfig,
1201
    ) -> bool:
1202
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1203
        return model_cls.is_pooling_model
1204
1205
1206

    def is_multimodal_model(
        self,
1207
        architectures: str | list[str],
1208
        model_config: ModelConfig,
1209
    ) -> bool:
1210
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1211
        return model_cls.supports_multimodal
1212

1213
    def is_multimodal_raw_input_only_model(
1214
        self,
1215
        architectures: str | list[str],
1216
        model_config: ModelConfig,
1217
    ) -> bool:
1218
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1219
        return model_cls.supports_multimodal_raw_input_only
1220

1221
1222
    def is_pp_supported_model(
        self,
1223
        architectures: str | list[str],
1224
        model_config: ModelConfig,
1225
    ) -> bool:
1226
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1227
        return model_cls.supports_pp
1228

1229
1230
    def model_has_inner_state(
        self,
1231
        architectures: str | list[str],
1232
        model_config: ModelConfig,
1233
    ) -> bool:
1234
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1235
        return model_cls.has_inner_state
1236

1237
1238
    def is_attention_free_model(
        self,
1239
        architectures: str | list[str],
1240
        model_config: ModelConfig,
1241
    ) -> bool:
1242
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1243
        return model_cls.is_attention_free
1244

1245
1246
    def is_hybrid_model(
        self,
1247
        architectures: str | list[str],
1248
        model_config: ModelConfig,
1249
    ) -> bool:
1250
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1251
1252
        return model_cls.is_hybrid

1253
1254
    def is_noops_model(
        self,
1255
        architectures: str | list[str],
1256
        model_config: ModelConfig,
1257
    ) -> bool:
1258
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1259
1260
        return model_cls.has_noops

1261
1262
    def is_transcription_model(
        self,
1263
        architectures: str | list[str],
1264
        model_config: ModelConfig,
1265
    ) -> bool:
1266
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1267
1268
        return model_cls.supports_transcription

1269
1270
    def is_transcription_only_model(
        self,
1271
        architectures: str | list[str],
1272
        model_config: ModelConfig,
1273
    ) -> bool:
1274
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1275
1276
        return model_cls.supports_transcription_only

1277

1278
1279
1280
1281
1282
1283
1284
1285
1286
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1287
1288
1289
1290
1291

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1292
1293
1294
1295
1296
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1297
        # `cloudpickle` allows pickling lambda functions directly
1298
        import cloudpickle
1299

1300
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1301
1302
1303

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1304
1305
1306
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1307
1308
1309
1310
1311
1312

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1313
1314
1315
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1316

1317
        with open(output_filepath, "rb") as f:
1318
1319
1320
1321
1322
1323
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1324

1325
1326
1327
1328
1329
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1330
1331
1332

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1333
1334
1335


if __name__ == "__main__":
1336
    _run()