registry.py 52.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.tasks import ScoreType
34
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
35
from vllm.utils.hashing import safe_hash
36

37
38
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
39
    from vllm.config.pooler import SequencePoolingType, TokenPoolingType
40
41
else:
    AttnTypeStr = Any
42
43
    SequencePoolingType = Any
    TokenPoolingType = Any
44
45


46
47
48
49
50
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
51
    requires_raw_input_tokens,
52
    supports_mamba_prefix_caching,
53
54
55
56
57
58
59
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
60
    get_attn_type,
61
62
    get_default_seq_pooling_type,
    get_default_tok_pooling_type,
63
    get_score_type,
64
65
66
    is_pooling_model,
    is_text_generation_model,
)
67
68
69

logger = init_logger(__name__)

70
71
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
72
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
73
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
74
75
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
76
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
77
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
78
    "AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
79
80
81
82
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
83
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
84
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
85
    "BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
86
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
87
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
88
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
89
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
90
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
91
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
92
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
93
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
94
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
95
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
96
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
97
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
98
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
99
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
100
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
101
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
102
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
103
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
Kyungmin Lee's avatar
Kyungmin Lee committed
104
    "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
105
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
106
107
108
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
109
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
110
111
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
112
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
113
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
114
    "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
115
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
116
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
117
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
118
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
119
    "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
Jee Jee Li's avatar
Jee Jee Li committed
120
    "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
121
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
122
123
124
125
126
127
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
128
129
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
130
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
131
132
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
133
134
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
135
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
136
    "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
137
    "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
138
139
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
140
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
141
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
142
143
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
144
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
145
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
146
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
147
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),
148
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
149
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
150
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
151
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
152
153
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
154
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
155
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
156
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
157
158
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
159
160
161
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
162
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
163
    "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
164
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
165
166
167
168
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
169
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
170
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
171
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
172
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
173
    "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
174
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
175
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
176
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
177
    "OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
178
179
180
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
181
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
182
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
183
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
184
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
185
186
187
188
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
189
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
190
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
191
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
192
193
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
194
195
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
196
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
197
198
    "SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
    "SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
199
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Li Xie's avatar
Li Xie committed
200
    "Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
Song's avatar
Song committed
201
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
csy0225's avatar
csy0225 committed
202
    "Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
203
204
205
206
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
207
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
208
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
209
    "TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
210
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
211
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
212
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
213
214
215
}

_EMBEDDING_MODELS = {
216
    # [Text-only]
217
    "BertModel": ("bert", "BertEmbeddingModel"),
218
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
219
    "ErnieModel": ("ernie", "ErnieEmbeddingModel"),
220
    "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
221
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
222
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
223
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
224
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
225
    "GritLM": ("gritlm", "GritLM"),
226
227
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
228
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
229
    "LlamaModel": ("llama", "LlamaForCausalLM"),
230
231
    **{
        # Multiple models share the same architecture, so we include them all
232
233
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
234
235
        if arch == "LlamaForCausalLM"
    },
236
    "MistralModel": ("llama", "LlamaForCausalLM"),
237
    "ModernBertModel": ("modernbert", "ModernBertModel"),
238
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
239
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
240
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
241
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
242
243
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
244
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
245
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
chengchengpei's avatar
chengchengpei committed
246
247
248
249
    "VoyageQwen3BidirectionalEmbedModel": (
        "voyage",
        "VoyageQwen3BidirectionalEmbedModel",
    ),
250
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
251
    # [Multimodal]
252
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
253
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
254
    "LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
255
256
257
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
258
    ),
259
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
260
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
261
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
262
263
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
264
    # models for the time being.
265
266
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
267
268
}

269
270
271
272
273
_LATE_INTERACTION_MODELS = {
    # [Text-only]
    "HF_ColBERT": ("colbert", "ColBERTModel"),
    "ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
    "ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
274
    "ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
275
276
    # [Multimodal]
    "ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
277
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
278
279
    "ColQwen3": ("colqwen3", "ColQwen3Model"),
    "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
280
    "ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
281
282
283
284
285
286
287
288
289
290
    "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
}

_REWARD_MODELS = {
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
}

_TOKEN_CLASSIFICATION_MODELS = {
291
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
292
    "ErnieForTokenClassification": ("ernie", "ErnieForTokenClassification"),
293
294
295
296
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
297
298
299
300
    "Qwen3ASRForcedAlignerForTokenClassification": (
        "qwen3_asr_forced_aligner",
        "Qwen3ASRForcedAlignerForTokenClassification",
    ),
301
302
303
304
305
}

_SEQUENCE_CLASSIFICATION_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
306
    "ErnieForSequenceClassification": ("ernie", "ErnieForSequenceClassification"),
307
308
309
310
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
311
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),
312
313
314
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
315
    ),
316
317
318
319
320
321
322
323
324
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
325
326
327
328
329
330
    # [Multimodal]
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaNemotronVLForSequenceClassification": (
        "nemotron_vl",
        "LlamaNemotronVLForSequenceClassification",
    ),
331
332
}

333
_MULTIMODAL_MODELS = {
334
    # [Decoder-only]
335
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
336
337
338
339
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
340
341
342
343
    "MusicFlamingoForConditionalGeneration": (
        "musicflamingo",
        "MusicFlamingoForConditionalGeneration",
    ),
344
345
346
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
347
    ),
348
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
349
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
350
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
351
352
353
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
354
    ),
355
356
    "Cheers": ("cheers", "CheersForConditionalGeneration"),
    "CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
357
358
359
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
360
    ),
361
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
362
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
RED's avatar
RED committed
363
    "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
364
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
365
366
367
368
    "Eagle2_5_VLForConditionalGeneration": (
        "eagle2_5_vl",
        "Eagle2_5_VLForConditionalGeneration",
    ),
369
370
371
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
372
    ),
373
374
375
376
    "FireRedASR2ForConditionalGeneration": (
        "fireredasr2",
        "FireRedASR2ForConditionalGeneration",
    ),
377
    "FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
378
379
380
381
    "FunAudioChatForConditionalGeneration": (
        "funaudiochat",
        "FunAudioChatForConditionalGeneration",
    ),
382
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
383
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),
384
385
386
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
387
    ),
388
    "Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
389
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
390
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
391
392
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
393
    "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
394
395
396
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
397
    ),
398
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
399
400
401
402
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
403
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
404
405
406
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
407
    ),
408
409
410
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
411
    ),
zxy's avatar
zxy committed
412
413
414
415
    "InternS1ProForConditionalGeneration": (
        "interns1_pro",
        "InternS1ProForConditionalGeneration",
    ),
416
417
418
419
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
420
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
421
    "KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
422
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
423
424
425
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
426
    ),
427
428
429
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
    "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
    "MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
430
431
432
433
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
434
    "Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
435
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),
436
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
437
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
438
439
440
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
441
    ),
442
443
444
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
445
    ),
446
447
448
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
449
    ),
450
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
451
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
452
453
454
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
455
    ),
456
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
457
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
458
459
460
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
461
    ),
462
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
463
    "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
464
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
465
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
466
    "OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
467
468
469
470
    "OpenPanguVLForConditionalGeneration": (
        "openpangu_vl",
        "OpenPanguVLForConditionalGeneration",
    ),
471
    "Ovis": ("ovis", "Ovis"),
472
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
473
474
    "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
    "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
475
476
477
478
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
479
480
481
482
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
483
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
484
    "Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
485
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
486
487
488
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
489
490
491
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
492
    ),
493
494
495
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
496
    ),
497
498
499
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
500
    ),
501
502
503
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
504
    ),
505
506
507
508
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
Roger Wang's avatar
Roger Wang committed
509
510
511
512
    "Qwen3ASRForConditionalGeneration": (
        "qwen3_asr",
        "Qwen3ASRForConditionalGeneration",
    ),
513
514
    "Qwen3ASRRealtimeGeneration": ("qwen3_asr_realtime", "Qwen3ASRRealtimeGeneration"),
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
515
516
517
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
518
    ),
519
    "Qwen3_5ForConditionalGeneration": ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
520
521
522
523
    "Qwen3_5MoeForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5MoeForConditionalGeneration",
    ),
524
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
525
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
526
527
528
529
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
    "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
530
531
532
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
533
    ),
534
    "UltravoxModel": ("ultravox", "UltravoxModel"),
535
536
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
    "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
537
    # [Encoder-decoder]
538
    "CohereAsrForConditionalGeneration": (
Ekagra Ranjan's avatar
Ekagra Ranjan committed
539
        "cohere_asr",
540
        "CohereAsrForConditionalGeneration",
Ekagra Ranjan's avatar
Ekagra Ranjan committed
541
    ),
542
543
544
545
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
546
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),
547
}
548
549

_SPECULATIVE_DECODING_MODELS = {
550
    "ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
551
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
552
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
553
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
554
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
555
    "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
556
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
557
    "Eagle3MiniMaxM2ForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
558
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
559
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
560
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
561
562
563
564
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
565
566
    "Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
    "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
567
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
568
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
569
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
570
    "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
571
    "NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
XuruiYang's avatar
XuruiYang committed
572
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
573
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
574
    "Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
575
    "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
576
    "MedusaModel": ("medusa", "Medusa"),
577
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
578
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
csy0225's avatar
csy0225 committed
579
    "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
580
581
    "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
    "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
582
583
584
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
585
}
586

587
_TRANSFORMERS_SUPPORTED_MODELS = {
588
589
590
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
591
592
593
594
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
595
596
597
}

_TRANSFORMERS_BACKEND_MODELS = {
598
    # Text generation models
599
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
618
    "TransformersForSequenceClassification": (
619
        "transformers",
620
        "TransformersForSequenceClassification",
621
    ),
622
    "TransformersMoEForSequenceClassification": (
623
        "transformers",
624
        "TransformersMoEForSequenceClassification",
625
    ),
626
627
628
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
629
    ),
630
}
631

632
_VLLM_MODELS = {
633
    **_TEXT_GENERATION_MODELS,
634
    **_EMBEDDING_MODELS,
635
636
637
638
    **_LATE_INTERACTION_MODELS,
    **_REWARD_MODELS,
    **_TOKEN_CLASSIFICATION_MODELS,
    **_SEQUENCE_CLASSIFICATION_MODELS,
639
    **_MULTIMODAL_MODELS,
640
    **_SPECULATIVE_DECODING_MODELS,
641
642
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
643
644
}

645
646
647
648
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
649
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
650

651
_PREVIOUSLY_SUPPORTED_MODELS = {
652
    "MotifForCausalLM": "0.10.2",
653
    "Phi3SmallForCausalLM": "0.9.2",
654
    "Phi4FlashForCausalLM": "0.10.2",
655
    "Phi4MultimodalForCausalLM": "0.12.0",
656
657
658
659
660
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "DonutForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
661

662
663
664
665
666
667
668
_OOT_SUPPORTED_MODELS = {
    "BartModel": "https://github.com/vllm-project/bart-plugin",
    "BartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "Florence2ForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
    "MBartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
}

669

670
671
@dataclass(frozen=True)
class _ModelInfo:
672
    architecture: str
673
    is_text_generation_model: bool
674
    is_pooling_model: bool
675
    attn_type: AttnTypeStr
676
677
    default_seq_pooling_type: SequencePoolingType
    default_tok_pooling_type: TokenPoolingType
678
    score_type: ScoreType
679
    supports_multimodal: bool
680
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
681
    requires_raw_input_tokens: bool
682
    supports_multimodal_encoder_tp_data: bool
683
    supports_pp: bool
684
685
    has_inner_state: bool
    is_attention_free: bool
686
    is_hybrid: bool
687
    has_noops: bool
688
    supports_mamba_prefix_caching: bool
689
    supports_transcription: bool
690
    supports_transcription_only: bool
691
692

    @staticmethod
693
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
694
        return _ModelInfo(
695
            architecture=model.__name__,
696
            is_text_generation_model=is_text_generation_model(model),
697
            is_pooling_model=is_pooling_model(model),
698
699
            default_seq_pooling_type=get_default_seq_pooling_type(model),
            default_tok_pooling_type=get_default_tok_pooling_type(model),
700
            attn_type=get_attn_type(model),
701
            score_type=get_score_type(model),
702
            supports_multimodal=supports_multimodal(model),
703
704
705
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
706
            requires_raw_input_tokens=requires_raw_input_tokens(model),
707
708
709
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
710
            supports_pp=supports_pp(model),
711
712
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
713
            is_hybrid=is_hybrid(model),
714
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
715
            supports_transcription=supports_transcription(model),
716
717
718
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
719
            has_noops=has_noops(model),
720
        )
721
722


723
724
725
726
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
727

728
    @abstractmethod
729
    def load_model_cls(self) -> type[nn.Module]:
730
        raise NotImplementedError
731
732


733
734
735
736
737
738
739
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
740
    model_cls: type[nn.Module]
741
742

    @staticmethod
743
    def from_model_cls(model_cls: type[nn.Module]):
744
745
746
747
748
749
750
751
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

752
    def load_model_cls(self) -> type[nn.Module]:
753
754
755
756
757
758
759
760
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
761

762
763
764
    module_name: str
    class_name: str

765
766
767
768
769
770
771
772
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

773
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
774
775
        try:
            try:
776
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
777
778
779
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
780
                logger.debug(
781
                    "Cached model info file for class %s.%s not found",
782
783
784
                    self.module_name,
                    self.class_name,
                )
785
786
787
                return None

            if mi_dict["hash"] != module_hash:
788
                logger.debug(
789
                    "Cached model info file for class %s.%s is stale",
790
791
792
                    self.module_name,
                    self.class_name,
                )
793
794
795
796
797
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
798
            logger.debug(
799
                "Cached model info for class %s.%s error. ",
800
801
802
                self.module_name,
                self.class_name,
            )
803
804
            return None

805
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
806
807
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
808

809
810
811
812
813
814
815
816
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
817
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
818
819
820
821
822
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
823
    def inspect_model_cls(self) -> _ModelInfo:
824
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
825
        module_hash = None
826

827
828
        if model_path.exists():
            with open(model_path, "rb") as f:
829
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
830
831
832

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
833
                logger.debug(
834
                    "Loaded model info for class %s.%s from cache",
835
836
837
                    self.module_name,
                    self.class_name,
                )
838
839
                return mi
            else:
840
                logger.debug(
841
                    "Cache model info for class %s.%s miss. Loading model instead.",
842
843
844
                    self.module_name,
                    self.class_name,
                )
845
846
847

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
848
849
850
851
852
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
853
854

        # save cache file
855
856
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
857
858

        return mi
859

860
    def load_model_cls(self) -> type[nn.Module]:
861
862
863
864
865
866
867
868
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
869
) -> type[nn.Module] | None:
870
    from vllm.platforms import current_platform
871

872
    current_platform.verify_model_arch(model_arch)
873
874
875
    try:
        return model.load_model_cls()
    except Exception:
876
        logger.exception("Error in loading model architecture '%s'", model_arch)
877
        return None
878
879


880
881
882
883
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
884
) -> _ModelInfo | None:
885
886
887
    try:
        return model.inspect_model_cls()
    except Exception:
888
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
889
        return None
890
891


892
893
894
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
895
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
896

897
    def get_supported_archs(self) -> Set[str]:
898
        return self.models.keys()
899

900
901
902
    def register_model(
        self,
        model_arch: str,
903
        model_cls: type[nn.Module] | str,
904
    ) -> None:
905
906
907
        """
        Register an external model to be used in vLLM.

908
        `model_cls` can be either:
909

910
        - A [`torch.nn.Module`][] class directly referencing the model.
911
        - A string in the format `<module>:<class>` which can be used to
912
913
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
914
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
915
        """
916
917
918
919
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

920
        if model_arch in self.models:
921
922
            logger.warning(
                "Model architecture %s is already registered, and will be "
923
924
925
926
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
927
928
929
930
931
932

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
933

934
            model = _LazyRegisteredModel(*split_str)
935
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
936
            model = _RegisteredModel.from_model_cls(model_cls)
937
        else:
938
939
940
941
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
942
            raise TypeError(msg)
943

944
        self.models[model_arch] = model
945

946
    def _raise_for_unsupported(self, architectures: list[str]):
947
        all_supported_archs = self.get_supported_archs()
948

949
950
951
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
952
953
                "to be inspected. Please check the logs for more details."
            )
954

955
956
957
958
959
960
961
962
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
963
964
                    "use this model architecture."
                )
965
966
967
968
969
970
971
972
            if arch in _OOT_SUPPORTED_MODELS:
                plugin_url = _OOT_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} is not supported in-tree anymore. "
                    f"Please install the plugin at {plugin_url} if you want to "
                    "use this model architecture."
                )
973

974
975
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
976
977
            f"Supported architectures: {all_supported_archs}"
        )
978

979
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
980
981
        if model_arch not in self.models:
            return None
982

983
        return _try_load_model_cls(model_arch, self.models[model_arch])
984

985
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
986
987
        if model_arch not in self.models:
            return None
988

989
990
991
992
993
994
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
995
    ) -> str | None:
996
997
998
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

999
1000
1001
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1018
                        trust_remote_code=model_config.trust_remote_code,
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1031
                        trust_remote_code=model_config.trust_remote_code,
1032
1033
1034
1035
1036
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
1037
                if model_config.model_impl != "transformers":
1038
1039
1040
1041
1042
1043
1044
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
1045
1046
                    "'auto_map' (relevant if the model is custom)."
                )
1047
1048

        if not model_module.is_backend_compatible():
1049
            if model_config.model_impl != "transformers":
1050
                return None
1051

1052
1053
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
1054
1055
                "is not compatible with vLLM."
            )
1056

1057
        return model_config._get_transformers_backend_cls()
1058

1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
1084

1085
1086
    def inspect_model_cls(
        self,
1087
        architectures: str | list[str],
1088
        model_config: ModelConfig,
1089
    ) -> tuple[_ModelInfo, str]:
1090
1091
        if isinstance(architectures, str):
            architectures = [architectures]
1092
1093
        if not architectures:
            raise ValueError("No model architectures are specified")
1094
1095

        # Require transformers impl
1096
        if model_config.model_impl == "transformers":
1097
            arch = self._try_resolve_transformers(architectures[0], model_config)
1098
1099
1100
1101
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
1102
        elif model_config.model_impl == "terratorch":
1103
1104
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
1105

1106
        # Fallback to transformers impl (after resolving convert_type)
1107
1108
1109
1110
1111
1112
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1113
1114
1115
1116
1117
1118
1119
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1120
            model_info = self._try_inspect_model_cls(normalized_arch)
1121
            if model_info is not None:
1122
                return (model_info, arch)
1123

1124
        # Fallback to transformers impl (before resolving runner_type)
1125
1126
1127
1128
1129
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1130
1131
1132
1133
1134
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

1135
        return self._raise_for_unsupported(architectures)
1136

1137
1138
    def resolve_model_cls(
        self,
1139
        architectures: str | list[str],
1140
        model_config: ModelConfig,
1141
    ) -> tuple[type[nn.Module], str]:
1142
1143
        if isinstance(architectures, str):
            architectures = [architectures]
1144
1145
        if not architectures:
            raise ValueError("No model architectures are specified")
1146
1147

        # Require transformers impl
1148
        if model_config.model_impl == "transformers":
1149
            arch = self._try_resolve_transformers(architectures[0], model_config)
1150
1151
1152
1153
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1154
        elif model_config.model_impl == "terratorch":
1155
1156
1157
1158
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1159

1160
        # Fallback to transformers impl (after resolving convert_type)
1161
1162
1163
1164
1165
1166
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1167
1168
1169
1170
1171
1172
1173
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1174
            model_cls = self._try_load_model_cls(normalized_arch)
1175
1176
            if model_cls is not None:
                return (model_cls, arch)
1177

1178
        # Fallback to transformers impl (before resolving runner_type)
1179
1180
1181
1182
1183
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1184
1185
1186
1187
1188
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1189
        return self._raise_for_unsupported(architectures)
1190

1191
1192
    def is_text_generation_model(
        self,
1193
        architectures: str | list[str],
1194
        model_config: ModelConfig,
1195
    ) -> bool:
1196
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1197
        return model_cls.is_text_generation_model
1198

1199
    def is_pooling_model(
1200
        self,
1201
        architectures: str | list[str],
1202
        model_config: ModelConfig,
1203
    ) -> bool:
1204
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1205
        return model_cls.is_pooling_model
1206
1207
1208

    def is_multimodal_model(
        self,
1209
        architectures: str | list[str],
1210
        model_config: ModelConfig,
1211
    ) -> bool:
1212
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1213
        return model_cls.supports_multimodal
1214

1215
    def is_multimodal_raw_input_only_model(
1216
        self,
1217
        architectures: str | list[str],
1218
        model_config: ModelConfig,
1219
    ) -> bool:
1220
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1221
        return model_cls.supports_multimodal_raw_input_only
1222

1223
1224
    def is_pp_supported_model(
        self,
1225
        architectures: str | list[str],
1226
        model_config: ModelConfig,
1227
    ) -> bool:
1228
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1229
        return model_cls.supports_pp
1230

1231
1232
    def model_has_inner_state(
        self,
1233
        architectures: str | list[str],
1234
        model_config: ModelConfig,
1235
    ) -> bool:
1236
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1237
        return model_cls.has_inner_state
1238

1239
1240
    def is_attention_free_model(
        self,
1241
        architectures: str | list[str],
1242
        model_config: ModelConfig,
1243
    ) -> bool:
1244
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1245
        return model_cls.is_attention_free
1246

1247
1248
    def is_hybrid_model(
        self,
1249
        architectures: str | list[str],
1250
        model_config: ModelConfig,
1251
    ) -> bool:
1252
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1253
1254
        return model_cls.is_hybrid

1255
1256
    def is_noops_model(
        self,
1257
        architectures: str | list[str],
1258
        model_config: ModelConfig,
1259
    ) -> bool:
1260
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1261
1262
        return model_cls.has_noops

1263
1264
    def is_transcription_model(
        self,
1265
        architectures: str | list[str],
1266
        model_config: ModelConfig,
1267
    ) -> bool:
1268
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1269
1270
        return model_cls.supports_transcription

1271
1272
    def is_transcription_only_model(
        self,
1273
        architectures: str | list[str],
1274
        model_config: ModelConfig,
1275
    ) -> bool:
1276
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1277
1278
        return model_cls.supports_transcription_only

1279

1280
1281
1282
1283
1284
1285
1286
1287
1288
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1289
1290
1291
1292
1293

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1294
1295
1296
1297
1298
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1299
        # `cloudpickle` allows pickling lambda functions directly
1300
        import cloudpickle
1301

1302
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1303
1304
1305

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1306
1307
1308
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1309
1310
1311
1312
1313
1314

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1315
1316
1317
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1318

1319
        with open(output_filepath, "rb") as f:
1320
1321
1322
1323
1324
1325
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1326

1327
1328
1329
1330
1331
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1332
1333
1334

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1335
1336
1337


if __name__ == "__main__":
1338
    _run()