registry.py 53.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.tasks import ScoreType
34
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
35
from vllm.utils.hashing import safe_hash
36

37
38
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
39
    from vllm.config.pooler import SequencePoolingType, TokenPoolingType
40
41
else:
    AttnTypeStr = Any
42
43
    SequencePoolingType = Any
    TokenPoolingType = Any
44
45


46
47
48
49
50
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
51
    requires_raw_input_tokens,
52
    supports_mamba_prefix_caching,
53
54
55
56
57
58
59
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
60
    get_attn_type,
61
62
    get_default_seq_pooling_type,
    get_default_tok_pooling_type,
63
    get_score_type,
64
65
66
    is_pooling_model,
    is_text_generation_model,
)
67
68
69

logger = init_logger(__name__)

70
71
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
72
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
73
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
74
75
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
76
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
77
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
78
    "AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
79
80
81
82
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
83
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
84
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
85
    "BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
86
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
87
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
88
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
89
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
90
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
91
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
92
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
93
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
94
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
95
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
96
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
97
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
98
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
99
    "DeepseekV4ForCausalLM": ("deepseek_v4", "DeepseekV4ForCausalLM"),
100
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
101
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
102
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
103
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
104
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
Kyungmin Lee's avatar
Kyungmin Lee committed
105
    "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
106
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
107
108
109
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
110
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
111
112
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
113
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
114
    "Rnj1ForCausalLM": ("rnj1", "Rnj1ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
115
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
116
    "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
117
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
118
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
119
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
120
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
121
    "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
Jee Jee Li's avatar
Jee Jee Li committed
122
    "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
123
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
124
125
126
127
128
129
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
130
131
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
132
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
133
134
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
135
136
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
137
    "HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
138
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
139
    "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
140
    "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
141
142
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
143
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
144
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
145
146
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
147
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
148
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
149
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
150
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
151
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
152
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
153
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
154
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
155
156
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
157
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
158
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
159
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
160
161
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
162
163
164
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
165
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
166
    "Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
167
    "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
168
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
169
170
171
172
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
173
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
174
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
175
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
176
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
177
    "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
178
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
179
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
180
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
181
    "OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
182
183
184
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
185
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
186
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
187
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
188
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
189
    "Param2MoEForCausalLM": ("param2moe", "Param2MoEForCausalLM"),
190
191
192
193
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
194
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
195
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
196
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
197
198
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
199
200
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
201
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
202
203
    "SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
    "SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
204
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Li Xie's avatar
Li Xie committed
205
    "Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
Song's avatar
Song committed
206
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
csy0225's avatar
csy0225 committed
207
    "Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
208
209
210
211
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
212
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
213
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
214
    "TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
215
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
216
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
217
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
218
219
220
}

_EMBEDDING_MODELS = {
221
    # [Text-only]
222
    "BertModel": ("bert", "BertEmbeddingModel"),
223
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
224
    "ErnieModel": ("ernie", "ErnieEmbeddingModel"),
225
    "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
226
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
227
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
228
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
229
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
230
    "GritLM": ("gritlm", "GritLM"),
231
232
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
233
    "JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
234
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
235
    "LlamaModel": ("llama", "LlamaForCausalLM"),
236
237
    **{
        # Multiple models share the same architecture, so we include them all
238
239
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
240
241
        if arch == "LlamaForCausalLM"
    },
242
    "MistralModel": ("llama", "LlamaForCausalLM"),
243
    "ModernBertModel": ("modernbert", "ModernBertModel"),
244
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
245
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
246
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
247
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
248
249
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
250
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
251
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
chengchengpei's avatar
chengchengpei committed
252
253
254
255
    "VoyageQwen3BidirectionalEmbedModel": (
        "voyage",
        "VoyageQwen3BidirectionalEmbedModel",
    ),
256
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
257
    # [Multimodal]
258
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
259
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
260
    "LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
261
262
263
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
264
    ),
265
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
266
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
267
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
268
269
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
270
    # models for the time being.
271
272
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
273
274
}

275
276
277
278
279
_LATE_INTERACTION_MODELS = {
    # [Text-only]
    "HF_ColBERT": ("colbert", "ColBERTModel"),
    "ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
    "ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
280
    "ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
281
    "JinaForRanking": ("jina", "JinaForRanking"),
282
283
    # [Multimodal]
    "ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
284
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
285
286
    "ColQwen3": ("colqwen3", "ColQwen3Model"),
    "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
287
    "ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
288
289
290
291
292
293
294
295
296
297
    "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
}

_REWARD_MODELS = {
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
}

_TOKEN_CLASSIFICATION_MODELS = {
298
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
299
    "ErnieForTokenClassification": ("ernie", "ErnieForTokenClassification"),
300
301
302
303
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
304
305
306
307
    "Qwen3ASRForcedAlignerForTokenClassification": (
        "qwen3_asr_forced_aligner",
        "Qwen3ASRForcedAlignerForTokenClassification",
    ),
308
309
310
311
312
}

_SEQUENCE_CLASSIFICATION_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
313
    "ErnieForSequenceClassification": ("ernie", "ErnieForSequenceClassification"),
314
315
316
317
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
318
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),
319
320
321
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
322
    ),
323
324
325
326
327
328
329
330
331
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
332
333
334
335
336
337
    # [Multimodal]
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaNemotronVLForSequenceClassification": (
        "nemotron_vl",
        "LlamaNemotronVLForSequenceClassification",
    ),
338
339
}

340
_MULTIMODAL_MODELS = {
341
    # [Decoder-only]
342
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
343
344
345
346
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
347
348
349
350
    "MusicFlamingoForConditionalGeneration": (
        "musicflamingo",
        "MusicFlamingoForConditionalGeneration",
    ),
351
352
353
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
354
    ),
355
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
356
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
357
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
358
359
360
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
361
    ),
362
363
    "Cheers": ("cheers", "CheersForConditionalGeneration"),
    "CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
364
365
366
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
367
    ),
368
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
369
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
RED's avatar
RED committed
370
    "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
371
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
372
373
374
375
    "Eagle2_5_VLForConditionalGeneration": (
        "eagle2_5_vl",
        "Eagle2_5_VLForConditionalGeneration",
    ),
376
377
378
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
379
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
380
381
382
383
    "Exaone4_5_ForConditionalGeneration": (
        "exaone4_5",
        "Exaone4_5_ForConditionalGeneration",
    ),  # noqa: E501
384
385
386
387
    "FireRedASR2ForConditionalGeneration": (
        "fireredasr2",
        "FireRedASR2ForConditionalGeneration",
    ),
388
    "FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
389
390
391
392
    "FireRedLIDForConditionalGeneration": (
        "fireredlid",
        "FireRedLIDForConditionalGeneration",
    ),
393
394
395
396
    "FunAudioChatForConditionalGeneration": (
        "funaudiochat",
        "FunAudioChatForConditionalGeneration",
    ),
397
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
398
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),
399
400
401
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
402
    ),
403
    "Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
404
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
405
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
406
407
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
408
    "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
409
410
411
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
412
    ),
413
414
415
416
    "Granite4VisionForConditionalGeneration": (
        "granite4_vision",
        "Granite4VisionForConditionalGeneration",
    ),
417
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
418
419
420
421
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
422
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
423
424
425
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
426
    ),
427
428
429
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
430
    ),
zxy's avatar
zxy committed
431
432
433
434
    "InternS1ProForConditionalGeneration": (
        "interns1_pro",
        "InternS1ProForConditionalGeneration",
    ),
435
436
437
438
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
439
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
440
    "KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
441
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
442
443
444
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
445
    ),
446
447
448
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
    "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
    "MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
449
450
451
452
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
453
    "Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
454
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),
455
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
456
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
457
458
459
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
460
    ),
461
462
463
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
464
    ),
465
466
467
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
468
    ),
469
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
470
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
471
472
473
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
474
    ),
475
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
476
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
477
478
479
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
480
    ),
481
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
482
    "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
483
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
484
485
    "NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
    "NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
486
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
487
    "OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
488
489
490
491
    "OpenPanguVLForConditionalGeneration": (
        "openpangu_vl",
        "OpenPanguVLForConditionalGeneration",
    ),
492
    "Ovis": ("ovis", "Ovis"),
493
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
494
495
    "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
    "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
496
497
498
499
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
500
501
502
503
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
504
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
505
    "Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
506
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
507
508
509
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
510
511
512
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
513
    ),
514
515
516
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
517
    ),
518
519
520
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
521
    ),
522
523
524
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
525
    ),
526
527
528
529
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
Roger Wang's avatar
Roger Wang committed
530
531
532
533
    "Qwen3ASRForConditionalGeneration": (
        "qwen3_asr",
        "Qwen3ASRForConditionalGeneration",
    ),
534
535
    "Qwen3ASRRealtimeGeneration": ("qwen3_asr_realtime", "Qwen3ASRRealtimeGeneration"),
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
536
537
538
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
539
    ),
540
    "Qwen3_5ForConditionalGeneration": ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
541
542
543
544
    "Qwen3_5MoeForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5MoeForConditionalGeneration",
    ),
545
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
546
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
547
548
549
550
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
    "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
551
552
553
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
554
    ),
555
    "UltravoxModel": ("ultravox", "UltravoxModel"),
556
557
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
    "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
558
    # [Encoder-decoder]
559
    "CohereAsrForConditionalGeneration": (
Ekagra Ranjan's avatar
Ekagra Ranjan committed
560
        "cohere_asr",
561
        "CohereAsrForConditionalGeneration",
Ekagra Ranjan's avatar
Ekagra Ranjan committed
562
    ),
563
564
565
566
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
567
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),
568
}
569
570

_SPECULATIVE_DECODING_MODELS = {
571
    "ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
572
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
573
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
574
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
575
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
576
    "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
577
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
578
    "Eagle3MiniMaxM2ForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
579
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
580
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
581
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
582
583
584
585
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
586
587
    "Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
    "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
588
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
589
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
590
    "DeepSeekV4MTPModel": ("deepseek_v4_mtp", "DeepSeekV4MTP"),
591
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
592
    "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
593
    "Exaone4_5_MTP": ("exaone4_5_mtp", "Exaone4_5_MTP"),
594
    "NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
XuruiYang's avatar
XuruiYang committed
595
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
596
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
597
    "Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
598
    "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
599
    "MedusaModel": ("medusa", "Medusa"),
600
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
601
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
csy0225's avatar
csy0225 committed
602
    "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
603
604
    "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
    "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
605
    "HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
606
607
608
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
609
}
610

611
_TRANSFORMERS_SUPPORTED_MODELS = {
612
613
614
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
615
616
617
618
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
619
620
621
}

_TRANSFORMERS_BACKEND_MODELS = {
622
    # Text generation models
623
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
642
    "TransformersForSequenceClassification": (
643
        "transformers",
644
        "TransformersForSequenceClassification",
645
    ),
646
    "TransformersMoEForSequenceClassification": (
647
        "transformers",
648
        "TransformersMoEForSequenceClassification",
649
    ),
650
651
652
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
653
    ),
654
}
655

656
_VLLM_MODELS = {
657
    **_TEXT_GENERATION_MODELS,
658
    **_EMBEDDING_MODELS,
659
660
661
662
    **_LATE_INTERACTION_MODELS,
    **_REWARD_MODELS,
    **_TOKEN_CLASSIFICATION_MODELS,
    **_SEQUENCE_CLASSIFICATION_MODELS,
663
    **_MULTIMODAL_MODELS,
664
    **_SPECULATIVE_DECODING_MODELS,
665
666
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
667
668
}

669
670
671
672
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
673
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
674

675
_PREVIOUSLY_SUPPORTED_MODELS = {
676
    "MotifForCausalLM": "0.10.2",
677
    "Phi3SmallForCausalLM": "0.9.2",
678
    "Phi4FlashForCausalLM": "0.10.2",
679
    "Phi4MultimodalForCausalLM": "0.12.0",
680
681
682
683
684
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "DonutForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
685

686
687
688
689
690
691
692
_OOT_SUPPORTED_MODELS = {
    "BartModel": "https://github.com/vllm-project/bart-plugin",
    "BartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "Florence2ForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "MBartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
}

693

694
695
@dataclass(frozen=True)
class _ModelInfo:
696
    architecture: str
697
    is_text_generation_model: bool
698
    is_pooling_model: bool
699
    attn_type: AttnTypeStr
700
701
    default_seq_pooling_type: SequencePoolingType
    default_tok_pooling_type: TokenPoolingType
702
    score_type: ScoreType
703
    supports_multimodal: bool
704
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
705
    requires_raw_input_tokens: bool
706
    supports_multimodal_encoder_tp_data: bool
707
    supports_pp: bool
708
709
    has_inner_state: bool
    is_attention_free: bool
710
    is_hybrid: bool
711
    has_noops: bool
712
    supports_mamba_prefix_caching: bool
713
    supports_transcription: bool
714
    supports_transcription_only: bool
715
716

    @staticmethod
717
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
718
        return _ModelInfo(
719
            architecture=model.__name__,
720
            is_text_generation_model=is_text_generation_model(model),
721
            is_pooling_model=is_pooling_model(model),
722
723
            default_seq_pooling_type=get_default_seq_pooling_type(model),
            default_tok_pooling_type=get_default_tok_pooling_type(model),
724
            attn_type=get_attn_type(model),
725
            score_type=get_score_type(model),
726
            supports_multimodal=supports_multimodal(model),
727
728
729
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
730
            requires_raw_input_tokens=requires_raw_input_tokens(model),
731
732
733
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
734
            supports_pp=supports_pp(model),
735
736
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
737
            is_hybrid=is_hybrid(model),
738
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
739
            supports_transcription=supports_transcription(model),
740
741
742
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
743
            has_noops=has_noops(model),
744
        )
745
746


747
748
749
750
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
751

752
    @abstractmethod
753
    def load_model_cls(self) -> type[nn.Module]:
754
        raise NotImplementedError
755
756


757
758
759
760
761
762
763
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
764
    model_cls: type[nn.Module]
765
766

    @staticmethod
767
    def from_model_cls(model_cls: type[nn.Module]):
768
769
770
771
772
773
774
775
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

776
    def load_model_cls(self) -> type[nn.Module]:
777
778
779
780
781
782
783
784
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
785

786
787
788
    module_name: str
    class_name: str

789
790
791
792
793
794
795
796
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

797
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
798
799
        try:
            try:
800
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
801
802
803
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
804
                logger.debug(
805
                    "Cached model info file for class %s.%s not found",
806
807
808
                    self.module_name,
                    self.class_name,
                )
809
810
811
                return None

            if mi_dict["hash"] != module_hash:
812
                logger.debug(
813
                    "Cached model info file for class %s.%s is stale",
814
815
816
                    self.module_name,
                    self.class_name,
                )
817
818
819
820
821
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
822
            logger.debug(
823
                "Cached model info for class %s.%s error. ",
824
825
826
                self.module_name,
                self.class_name,
            )
827
828
            return None

829
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
830
831
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
832

833
834
835
836
837
838
839
840
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
841
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
842
843
844
845
846
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
847
    def inspect_model_cls(self) -> _ModelInfo:
848
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
849
        module_hash = None
850

851
852
        if model_path.exists():
            with open(model_path, "rb") as f:
853
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
854
855
856

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
857
                logger.debug(
858
                    "Loaded model info for class %s.%s from cache",
859
860
861
                    self.module_name,
                    self.class_name,
                )
862
863
                return mi
            else:
864
                logger.debug(
865
                    "Cache model info for class %s.%s miss. Loading model instead.",
866
867
868
                    self.module_name,
                    self.class_name,
                )
869
870
871

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
872
873
874
875
876
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
877
878

        # save cache file
879
880
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
881
882

        return mi
883

884
    def load_model_cls(self) -> type[nn.Module]:
885
886
887
888
889
890
891
892
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
893
) -> type[nn.Module] | None:
894
    from vllm.platforms import current_platform
895

896
    current_platform.verify_model_arch(model_arch)
897
898
899
    try:
        return model.load_model_cls()
    except Exception:
900
        logger.exception("Error in loading model architecture '%s'", model_arch)
901
        return None
902
903


904
905
906
907
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
908
) -> _ModelInfo | None:
909
910
911
    try:
        return model.inspect_model_cls()
    except Exception:
912
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
913
        return None
914
915


916
917
918
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
919
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
920

921
    def get_supported_archs(self) -> Set[str]:
922
        return self.models.keys()
923

924
925
926
    def register_model(
        self,
        model_arch: str,
927
        model_cls: type[nn.Module] | str,
928
    ) -> None:
929
930
931
        """
        Register an external model to be used in vLLM.

932
        `model_cls` can be either:
933

934
        - A [`torch.nn.Module`][] class directly referencing the model.
935
        - A string in the format `<module>:<class>` which can be used to
936
937
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
938
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
939
        """
940
941
942
943
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

944
        if model_arch in self.models:
945
946
            logger.warning(
                "Model architecture %s is already registered, and will be "
947
948
949
950
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
951
952
953
954
955
956

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
957

958
            model = _LazyRegisteredModel(*split_str)
959
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
960
            model = _RegisteredModel.from_model_cls(model_cls)
961
        else:
962
963
964
965
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
966
            raise TypeError(msg)
967

968
        self.models[model_arch] = model
969

970
    def _raise_for_unsupported(self, architectures: list[str]):
971
        all_supported_archs = self.get_supported_archs()
972

973
974
975
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
976
977
                "to be inspected. Please check the logs for more details."
            )
978

979
980
981
982
983
984
985
986
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
987
988
                    "use this model architecture."
                )
989
990
991
992
993
994
995
996
            if arch in _OOT_SUPPORTED_MODELS:
                plugin_url = _OOT_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} is not supported in-tree anymore. "
                    f"Please install the plugin at {plugin_url} if you want to "
                    "use this model architecture."
                )
997

998
999
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
1000
1001
            f"Supported architectures: {all_supported_archs}"
        )
1002

1003
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
1004
1005
        if model_arch not in self.models:
            return None
1006

1007
        return _try_load_model_cls(model_arch, self.models[model_arch])
1008

1009
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
1010
1011
        if model_arch not in self.models:
            return None
1012

1013
1014
1015
1016
1017
1018
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
1019
    ) -> str | None:
1020
1021
1022
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

1023
1024
1025
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1042
                        trust_remote_code=model_config.trust_remote_code,
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1055
                        trust_remote_code=model_config.trust_remote_code,
1056
1057
1058
1059
1060
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
1061
                if model_config.model_impl != "transformers":
1062
1063
1064
1065
1066
1067
1068
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
1069
1070
                    "'auto_map' (relevant if the model is custom)."
                )
1071
1072

        if not model_module.is_backend_compatible():
1073
            if model_config.model_impl != "transformers":
1074
                return None
1075

1076
1077
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
1078
1079
                "is not compatible with vLLM."
            )
1080

1081
        return model_config._get_transformers_backend_cls()
1082

1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
1108

1109
1110
    def inspect_model_cls(
        self,
1111
        architectures: str | list[str],
1112
        model_config: ModelConfig,
1113
    ) -> tuple[_ModelInfo, str]:
1114
1115
        if isinstance(architectures, str):
            architectures = [architectures]
1116
1117
        if not architectures:
            raise ValueError("No model architectures are specified")
1118
1119

        # Require transformers impl
1120
        if model_config.model_impl == "transformers":
1121
            arch = self._try_resolve_transformers(architectures[0], model_config)
1122
1123
1124
1125
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
1126
        elif model_config.model_impl == "terratorch":
1127
1128
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
1129

1130
        # Fallback to transformers impl (after resolving convert_type)
1131
1132
1133
1134
1135
1136
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1137
1138
1139
1140
1141
1142
1143
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1144
            model_info = self._try_inspect_model_cls(normalized_arch)
1145
            if model_info is not None:
1146
                return (model_info, arch)
1147

1148
        # Fallback to transformers impl (before resolving runner_type)
1149
1150
1151
1152
1153
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1154
1155
1156
1157
1158
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

1159
        return self._raise_for_unsupported(architectures)
1160

1161
1162
    def resolve_model_cls(
        self,
1163
        architectures: str | list[str],
1164
        model_config: ModelConfig,
1165
    ) -> tuple[type[nn.Module], str]:
1166
1167
        if isinstance(architectures, str):
            architectures = [architectures]
1168
1169
        if not architectures:
            raise ValueError("No model architectures are specified")
1170
1171

        # Require transformers impl
1172
        if model_config.model_impl == "transformers":
1173
            arch = self._try_resolve_transformers(architectures[0], model_config)
1174
1175
1176
1177
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1178
        elif model_config.model_impl == "terratorch":
1179
1180
1181
1182
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1183

1184
        # Fallback to transformers impl (after resolving convert_type)
1185
1186
1187
1188
1189
1190
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1191
1192
1193
1194
1195
1196
1197
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1198
            model_cls = self._try_load_model_cls(normalized_arch)
1199
1200
            if model_cls is not None:
                return (model_cls, arch)
1201

1202
        # Fallback to transformers impl (before resolving runner_type)
1203
1204
1205
1206
1207
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1208
1209
1210
1211
1212
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1213
        return self._raise_for_unsupported(architectures)
1214

1215
1216
    def is_text_generation_model(
        self,
1217
        architectures: str | list[str],
1218
        model_config: ModelConfig,
1219
    ) -> bool:
1220
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1221
        return model_cls.is_text_generation_model
1222

1223
    def is_pooling_model(
1224
        self,
1225
        architectures: str | list[str],
1226
        model_config: ModelConfig,
1227
    ) -> bool:
1228
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1229
        return model_cls.is_pooling_model
1230
1231
1232

    def is_multimodal_model(
        self,
1233
        architectures: str | list[str],
1234
        model_config: ModelConfig,
1235
    ) -> bool:
1236
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1237
        return model_cls.supports_multimodal
1238

1239
    def is_multimodal_raw_input_only_model(
1240
        self,
1241
        architectures: str | list[str],
1242
        model_config: ModelConfig,
1243
    ) -> bool:
1244
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1245
        return model_cls.supports_multimodal_raw_input_only
1246

1247
1248
    def is_pp_supported_model(
        self,
1249
        architectures: str | list[str],
1250
        model_config: ModelConfig,
1251
    ) -> bool:
1252
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1253
        return model_cls.supports_pp
1254

1255
1256
    def model_has_inner_state(
        self,
1257
        architectures: str | list[str],
1258
        model_config: ModelConfig,
1259
    ) -> bool:
1260
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1261
        return model_cls.has_inner_state
1262

1263
1264
    def is_attention_free_model(
        self,
1265
        architectures: str | list[str],
1266
        model_config: ModelConfig,
1267
    ) -> bool:
1268
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1269
        return model_cls.is_attention_free
1270

1271
1272
    def is_hybrid_model(
        self,
1273
        architectures: str | list[str],
1274
        model_config: ModelConfig,
1275
    ) -> bool:
1276
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1277
1278
        return model_cls.is_hybrid

1279
1280
    def is_noops_model(
        self,
1281
        architectures: str | list[str],
1282
        model_config: ModelConfig,
1283
    ) -> bool:
1284
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1285
1286
        return model_cls.has_noops

1287
1288
    def is_transcription_model(
        self,
1289
        architectures: str | list[str],
1290
        model_config: ModelConfig,
1291
    ) -> bool:
1292
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1293
1294
        return model_cls.supports_transcription

1295
1296
    def is_transcription_only_model(
        self,
1297
        architectures: str | list[str],
1298
        model_config: ModelConfig,
1299
    ) -> bool:
1300
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1301
1302
        return model_cls.supports_transcription_only

1303

1304
1305
1306
1307
1308
1309
1310
1311
1312
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1313
1314
1315
1316
1317

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1318
1319
1320
1321
1322
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1323
        # `cloudpickle` allows pickling lambda functions directly
1324
        import cloudpickle
1325

1326
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1327
1328
1329

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1330
1331
1332
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1333
1334
1335
1336
1337
1338

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1339
1340
1341
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1342

1343
        with open(output_filepath, "rb") as f:
1344
1345
1346
1347
1348
1349
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1350

1351
1352
1353
1354
1355
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1356
1357
1358

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1359
1360
1361


if __name__ == "__main__":
1362
    _run()