registry.py 38.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7
import importlib
8
import os
9
import pickle
10
11
import subprocess
import sys
12
import tempfile
13
from abc import ABC, abstractmethod
14
from collections.abc import Set
15
from dataclasses import dataclass, field
16
from functools import lru_cache
17
from typing import Callable, Optional, TypeVar, Union
18
19

import torch.nn as nn
20
import transformers
21

22
23
from vllm.config import (ModelConfig, ModelImpl, iter_architecture_defaults,
                         try_match_architecture_defaults)
24
from vllm.logger import init_logger
25
26
from vllm.transformers_utils.dynamic_module import (
    try_get_class_from_dynamic_module)
27

28
29
from .interfaces import (has_inner_state, has_noops, is_attention_free,
                         is_hybrid, supports_cross_encoding,
30
31
                         supports_multimodal,
                         supports_multimodal_encoder_tp_data,
32
                         supports_multimodal_raw_input_only, supports_pp,
33
                         supports_transcription, supports_v0_only)
34
35
from .interfaces_base import (get_default_pooling_type, is_pooling_model,
                              is_text_generation_model)
36
37
38

logger = init_logger(__name__)

39
# yapf: disable
40
41
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
42
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
43
44
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
45
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
46
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
47
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
48
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
49
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
50
51
52
53
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
54
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
55
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
56
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
57
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
58
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
59
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
60
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
61
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
62
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
63
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
64
65
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
66
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
67
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
68
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
69
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
70
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
71
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
72
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
73
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
74
75
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
76
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
77
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
78
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
79
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
80
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
81
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
82
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
83
84
85
86
87
88
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
89
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),   # noqa: E501
90
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
91
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
92
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
93
94
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
95
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
96
97
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
98
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
99
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
100
101
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
102
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
103
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
104
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),  # noqa: E501
105
106
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
107
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
108
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
Dhia Eddine Rhaiem's avatar
Dhia Eddine Rhaiem committed
109
    "FalconH1ForCausalLM":("falcon_h1", "FalconH1ForCausalLM"),
110
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
111
112
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
113
114
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
115
    "MotifForCausalLM": ("motif", "MotifForCausalLM"),
116
117
118
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
119
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
120
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
121
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
122
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
123
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
124
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
125
126
127
128
129
130
131
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
132
    "Phi4FlashForCausalLM": ("phi4flash", "Phi4FlashForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
133
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
134
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
135
136
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
137
138
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
139
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
140
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Song's avatar
Song committed
141
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
142
143
144
145
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
146
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
147
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
148
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
149
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
150
151
152
}

_EMBEDDING_MODELS = {
153
    # [Text-only]
154
    "BertModel": ("bert", "BertEmbeddingModel"),
155
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
156
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
157
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
158
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
159
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
160
    "GritLM": ("gritlm", "GritLM"),
161
162
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
163
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
164
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
165
    "LlamaModel": ("llama", "LlamaForCausalLM"),
166
167
168
169
170
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
171
    "MistralModel": ("llama", "LlamaForCausalLM"),
172
    "ModernBertModel": ("modernbert", "ModernBertModel"),
173
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
174
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
175
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
176
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
177
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
178
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
179
180
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
181
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
182
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
183
    # [Multimodal]
Cyrus Leung's avatar
Cyrus Leung committed
184
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
185
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
186
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
187
188
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
189
    # models for the time being.
190
191
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
192
193
}

194
195
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
196
197
198
199
    "GteNewForSequenceClassification": ("bert_with_rope",
                                        "GteNewForSequenceClassification"),
    "ModernBertForSequenceClassification": ("modernbert",
                                            "ModernBertForSequenceClassification"),
200
201
202
203
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
204
    # [Auto-converted (see adapters.py)]
205
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"), # noqa: E501,
206
207
}

208
_MULTIMODAL_MODELS = {
209
    # [Decoder-only]
210
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
Jennifer Zhao's avatar
Jennifer Zhao committed
211
    "AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"),  # noqa: E501
212
213
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
214
    "Cohere2VisionForConditionalGeneration": ("cohere2_vision", "Cohere2VisionForConditionalGeneration"),  # noqa: E501
215
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
216
    "Ernie4_5_VLMoeForConditionalGeneration": ("ernie45_vl", "Ernie4_5_VLMoeForConditionalGeneration"),  # noqa: E501
217
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
218
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
219
    "Gemma3nForConditionalGeneration": ("gemma3n_mm", "Gemma3nForConditionalGeneration"),    # noqa: E501
220
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
221
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),  # noqa: E501
Jee Jee Li's avatar
Jee Jee Li committed
222
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),  # noqa: E501
223
    "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"),  # noqa: E501
224
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
225
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
226
    "NemotronH_Nano_VL": ("nano_nemotron_vl", "NemotronH_Nano_VL"),
Lyu Han's avatar
Lyu Han committed
227
    "InternS1ForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
228
    "InternVLForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
229
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
230
    "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"),  # noqa: E501
231
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
232
    "KeyeVL1_5ForConditionalGeneration": ("keye_vl1_5", "KeyeVL1_5ForConditionalGeneration"), # noqa: E501
233
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
234
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
235
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
236
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
237
238
239
240
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
241
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
242
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
243
    "MiniMaxVL01ForConditionalGeneration": ("minimax_vl_01", "MiniMaxVL01ForConditionalGeneration"),  # noqa: E501
244
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
245
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
246
    "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"),  # noqa: E501
247
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
248
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
249
    "Ovis": ("ovis", "Ovis"),
250
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
251
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
252
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
253
254
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
    "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"),  # noqa: E501
255
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
256
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
257
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
258
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
259
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
260
    "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
261
    "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
262
    "UltravoxModel": ("ultravox", "UltravoxModel"),
263
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
264
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
265
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
266
    "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"),  # noqa: E501
Patrick von Platen's avatar
Patrick von Platen committed
267
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
268
    # [Encoder-decoder]
269
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
270
}
271
272

_SPECULATIVE_DECODING_MODELS = {
273
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
274
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
275
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
276
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
277
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
278
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
279
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
280
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
281
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
282
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
283
    "MedusaModel": ("medusa", "Medusa"),
284
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
285
286
287
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
288
}
289

290
_TRANSFORMERS_SUPPORTED_MODELS = {
291
292
293
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
294
295
296
297
    "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
}

_TRANSFORMERS_BACKEND_MODELS = {
298
    "TransformersModel": ("transformers", "TransformersModel"),
299
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
300
    "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
301
}
302
# yapf: enable
303

304
_VLLM_MODELS = {
305
    **_TEXT_GENERATION_MODELS,
306
    **_EMBEDDING_MODELS,
307
    **_CROSS_ENCODER_MODELS,
308
    **_MULTIMODAL_MODELS,
309
    **_SPECULATIVE_DECODING_MODELS,
310
311
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
312
313
}

314
315
316
317
318
319
320
321
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
_SUBPROCESS_COMMAND = [
    sys.executable, "-m", "vllm.model_executor.models.registry"
]

322
323
_PREVIOUSLY_SUPPORTED_MODELS = {"Phi3SmallForCausalLM": "0.9.2"}

324

325
326
@dataclass(frozen=True)
class _ModelInfo:
327
    architecture: str
328
    is_text_generation_model: bool
329
    is_pooling_model: bool
330
    default_pooling_type: str
331
    supports_cross_encoding: bool
332
    supports_multimodal: bool
333
    supports_multimodal_raw_input_only: bool
334
    supports_multimodal_encoder_tp_data: bool
335
    supports_pp: bool
336
337
    has_inner_state: bool
    is_attention_free: bool
338
    is_hybrid: bool
339
    has_noops: bool
340
    supports_transcription: bool
341
    supports_transcription_only: bool
342
    supports_v0_only: bool
343
344

    @staticmethod
345
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
346
        return _ModelInfo(
347
            architecture=model.__name__,
348
            is_text_generation_model=is_text_generation_model(model),
349
            is_pooling_model=is_pooling_model(model),
350
            default_pooling_type=get_default_pooling_type(model),
351
            supports_cross_encoding=supports_cross_encoding(model),
352
            supports_multimodal=supports_multimodal(model),
353
354
            supports_multimodal_raw_input_only=
            supports_multimodal_raw_input_only(model),
355
356
            supports_multimodal_encoder_tp_data=
            supports_multimodal_encoder_tp_data(model),
357
            supports_pp=supports_pp(model),
358
359
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
360
            is_hybrid=is_hybrid(model),
361
            supports_transcription=supports_transcription(model),
362
363
            supports_transcription_only=(supports_transcription(model) and
                                         model.supports_transcription_only),
364
            supports_v0_only=supports_v0_only(model),
365
            has_noops=has_noops(model),
366
        )
367
368


369
class _BaseRegisteredModel(ABC):
370

371
372
373
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
374

375
    @abstractmethod
376
    def load_model_cls(self) -> type[nn.Module]:
377
        raise NotImplementedError
378
379


380
381
382
383
384
385
386
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
387
    model_cls: type[nn.Module]
388
389

    @staticmethod
390
    def from_model_cls(model_cls: type[nn.Module]):
391
392
393
394
395
396
397
398
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

399
    def load_model_cls(self) -> type[nn.Module]:
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
    module_name: str
    class_name: str

    # Performed in another process to avoid initializing CUDA
    def inspect_model_cls(self) -> _ModelInfo:
        return _run_in_subprocess(
            lambda: _ModelInfo.from_model_cls(self.load_model_cls()))

416
    def load_model_cls(self) -> type[nn.Module]:
417
418
419
420
421
422
423
424
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
425
) -> Optional[type[nn.Module]]:
426
    from vllm.platforms import current_platform
427
    current_platform.verify_model_arch(model_arch)
428
429
430
431
432
433
    try:
        return model.load_model_cls()
    except Exception:
        logger.exception("Error in loading model architecture '%s'",
                         model_arch)
        return None
434
435


436
437
438
439
440
441
442
443
444
445
446
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
        logger.exception("Error in inspecting model architecture '%s'",
                         model_arch)
        return None
447
448


449
450
451
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
452
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
453

454
    def get_supported_archs(self) -> Set[str]:
455
        return self.models.keys()
456

457
458
459
    def register_model(
        self,
        model_arch: str,
460
        model_cls: Union[type[nn.Module], str],
461
    ) -> None:
462
463
464
        """
        Register an external model to be used in vLLM.

465
        `model_cls` can be either:
466

467
        - A [`torch.nn.Module`][] class directly referencing the model.
468
        - A string in the format `<module>:<class>` which can be used to
469
470
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
471
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
472
        """
473
474
475
476
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

477
        if model_arch in self.models:
478
479
480
            logger.warning(
                "Model architecture %s is already registered, and will be "
                "overwritten by the new model class %s.", model_arch,
481
482
483
484
485
486
487
                model_cls)

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
488

489
            model = _LazyRegisteredModel(*split_str)
490
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
491
            model = _RegisteredModel.from_model_cls(model_cls)
492
493
494
495
        else:
            msg = ("`model_cls` should be a string or PyTorch model class, "
                   f"not a {type(model_arch)}")
            raise TypeError(msg)
496

497
        self.models[model_arch] = model
498

499
    def _raise_for_unsupported(self, architectures: list[str]):
500
        all_supported_archs = self.get_supported_archs()
501

502
503
504
505
506
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
                "to be inspected. Please check the logs for more details.")

507
508
509
510
511
512
513
514
515
516
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
                    "use this model architecture.")

517
518
519
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
            f"Supported architectures: {all_supported_archs}")
520

521
    def _try_load_model_cls(self,
522
                            model_arch: str) -> Optional[type[nn.Module]]:
523
524
        if model_arch not in self.models:
            return None
525

526
        return _try_load_model_cls(model_arch, self.models[model_arch])
527

528
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
529
530
        if model_arch not in self.models:
            return None
531

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> Optional[str]:
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

        auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map",
                                           None) or dict()

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
                if model_config.model_impl != ModelImpl.TRANSFORMERS:
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
                    "'auto_map' (relevant if the model is custom).")

        if not model_module.is_backend_compatible():
            if model_config.model_impl != ModelImpl.TRANSFORMERS:
589
                return None
590

591
592
593
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
                "is not compatible with vLLM.")
594

595
        return model_config._get_transformers_backend_cls()
596

597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
622

623
624
    def inspect_model_cls(
        self,
625
        architectures: Union[str, list[str]],
626
        model_config: ModelConfig,
627
    ) -> tuple[_ModelInfo, str]:
628
629
        if isinstance(architectures, str):
            architectures = [architectures]
630
631
        if not architectures:
            raise ValueError("No model architectures are specified")
632
633
634
635
636
637
638
639
640

        # Require transformers impl
        if model_config.model_impl == ModelImpl.TRANSFORMERS:
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
641
642
643
        elif model_config.model_impl == ModelImpl.TERRATORCH:
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
644

645
646
647
648
649
650
651
652
653
654
655
656
657
        # Fallback to transformers impl (after resolving convert_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO
                and getattr(model_config, "convert_type", "none") == "none"):
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
658
            model_info = self._try_inspect_model_cls(normalized_arch)
659
            if model_info is not None:
660
                return (model_info, arch)
661

662
663
664
        # Fallback to transformers impl (before resolving runner_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO):
665
666
667
668
669
670
671
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

672
        return self._raise_for_unsupported(architectures)
673

674
675
    def resolve_model_cls(
        self,
676
        architectures: Union[str, list[str]],
677
        model_config: ModelConfig,
678
    ) -> tuple[type[nn.Module], str]:
679
680
        if isinstance(architectures, str):
            architectures = [architectures]
681
682
        if not architectures:
            raise ValueError("No model architectures are specified")
683
684
685
686
687
688
689
690
691

        # Require transformers impl
        if model_config.model_impl == ModelImpl.TRANSFORMERS:
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
692
693
694
695
696
        elif model_config.model_impl == ModelImpl.TERRATORCH:
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
697

698
699
700
701
702
703
704
705
706
707
708
709
710
        # Fallback to transformers impl (after resolving convert_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO
                and getattr(model_config, "convert_type", "none") == "none"):
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
711
            model_cls = self._try_load_model_cls(normalized_arch)
712
713
            if model_cls is not None:
                return (model_cls, arch)
714

715
716
717
        # Fallback to transformers impl (before resolving runner_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO):
718
719
720
721
722
723
724
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

725
        return self._raise_for_unsupported(architectures)
726

727
728
    def is_text_generation_model(
        self,
729
        architectures: Union[str, list[str]],
730
        model_config: ModelConfig,
731
    ) -> bool:
732
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
733
        return model_cls.is_text_generation_model
734

735
    def is_pooling_model(
736
        self,
737
        architectures: Union[str, list[str]],
738
        model_config: ModelConfig,
739
    ) -> bool:
740
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
741
        return model_cls.is_pooling_model
742

743
744
    def is_cross_encoder_model(
        self,
745
        architectures: Union[str, list[str]],
746
        model_config: ModelConfig,
747
    ) -> bool:
748
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
749
        return model_cls.supports_cross_encoding
750

751
752
    def is_multimodal_model(
        self,
753
        architectures: Union[str, list[str]],
754
        model_config: ModelConfig,
755
    ) -> bool:
756
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
757
        return model_cls.supports_multimodal
758

759
    def is_multimodal_raw_input_only_model(
760
761
        self,
        architectures: Union[str, list[str]],
762
        model_config: ModelConfig,
763
    ) -> bool:
764
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
765
        return model_cls.supports_multimodal_raw_input_only
766

767
768
    def is_pp_supported_model(
        self,
769
        architectures: Union[str, list[str]],
770
        model_config: ModelConfig,
771
    ) -> bool:
772
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
773
        return model_cls.supports_pp
774

775
776
    def model_has_inner_state(
        self,
777
        architectures: Union[str, list[str]],
778
        model_config: ModelConfig,
779
    ) -> bool:
780
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
781
        return model_cls.has_inner_state
782

783
784
    def is_attention_free_model(
        self,
785
        architectures: Union[str, list[str]],
786
        model_config: ModelConfig,
787
    ) -> bool:
788
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
789
        return model_cls.is_attention_free
790

791
792
    def is_hybrid_model(
        self,
793
        architectures: Union[str, list[str]],
794
        model_config: ModelConfig,
795
    ) -> bool:
796
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
797
798
        return model_cls.is_hybrid

799
800
    def is_noops_model(
        self,
801
        architectures: Union[str, list[str]],
802
        model_config: ModelConfig,
803
    ) -> bool:
804
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
805
806
        return model_cls.has_noops

807
808
    def is_transcription_model(
        self,
809
        architectures: Union[str, list[str]],
810
        model_config: ModelConfig,
811
    ) -> bool:
812
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
813
814
        return model_cls.supports_transcription

815
816
817
    def is_transcription_only_model(
        self,
        architectures: Union[str, list[str]],
818
        model_config: ModelConfig,
819
    ) -> bool:
820
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
821
822
        return model_cls.supports_transcription_only

823
824
    def is_v1_compatible(
        self,
825
        architectures: Union[str, list[str]],
826
        model_config: ModelConfig,
827
    ) -> bool:
828
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
829
830
        return not model_cls.supports_v0_only

831
832

ModelRegistry = _ModelRegistry({
833
834
    model_arch:
    _LazyRegisteredModel(
835
836
837
838
839
840
841
842
843
844
        module_name=f"vllm.model_executor.models.{mod_relname}",
        class_name=cls_name,
    )
    for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
})

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
845
846
847
848
849
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

850
        # `cloudpickle` allows pickling lambda functions directly
851
        import cloudpickle
852
        input_bytes = cloudpickle.dumps((fn, output_filepath))
853
854
855

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
856
857
858
        returned = subprocess.run(_SUBPROCESS_COMMAND,
                                  input=input_bytes,
                                  capture_output=True)
859
860
861
862
863
864
865
866
867

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
            raise RuntimeError(f"Error raised in subprocess:\n"
                               f"{returned.stderr.decode()}") from e

868
        with open(output_filepath, "rb") as f:
869
870
871
872
873
874
875
876
877
878
879
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
880
881
882

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
883
884
885


if __name__ == "__main__":
886
    _run()