registry.py 38.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7
import importlib
8
import os
9
import pickle
10
11
import subprocess
import sys
12
import tempfile
13
from abc import ABC, abstractmethod
14
from collections.abc import Set
15
from dataclasses import dataclass, field
16
from functools import lru_cache
17
from typing import Callable, Optional, TypeVar, Union
18
19

import torch.nn as nn
20
import transformers
21

22
23
from vllm.config import (ModelConfig, ModelImpl, iter_architecture_defaults,
                         try_match_architecture_defaults)
24
from vllm.logger import init_logger
25
26
from vllm.transformers_utils.dynamic_module import (
    try_get_class_from_dynamic_module)
27

28
29
from .interfaces import (has_inner_state, has_noops, is_attention_free,
                         is_hybrid, supports_cross_encoding,
30
31
                         supports_multimodal,
                         supports_multimodal_encoder_tp_data,
32
                         supports_multimodal_raw_input_only, supports_pp,
33
                         supports_transcription, supports_v0_only)
34
35
from .interfaces_base import (get_default_pooling_type, is_pooling_model,
                              is_text_generation_model)
36
37
38

logger = init_logger(__name__)

39
# yapf: disable
40
41
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
42
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
43
44
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
45
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
46
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
47
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
48
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
49
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
50
51
52
53
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
54
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
55
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
56
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
57
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
58
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
59
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
60
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
61
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
62
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
63
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
64
65
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
66
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
67
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
68
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
69
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
70
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
71
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
72
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
73
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
74
75
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
76
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
77
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
78
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
79
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
80
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
81
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
82
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
83
84
85
86
87
88
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
89
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),   # noqa: E501
90
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
91
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
92
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
93
94
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
95
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
96
97
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
98
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
99
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
100
101
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
102
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
103
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
104
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),  # noqa: E501
105
106
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
107
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
108
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
Dhia Eddine Rhaiem's avatar
Dhia Eddine Rhaiem committed
109
    "FalconH1ForCausalLM":("falcon_h1", "FalconH1ForCausalLM"),
110
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
111
112
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
113
114
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
115
    "MotifForCausalLM": ("motif", "MotifForCausalLM"),
116
117
118
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
119
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
120
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
121
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
122
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
123
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
124
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
125
126
127
128
129
130
131
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
132
    "Phi4FlashForCausalLM": ("phi4flash", "Phi4FlashForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
133
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
134
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
135
136
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
137
138
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
139
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
140
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Song's avatar
Song committed
141
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
142
143
144
145
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
146
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
147
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
148
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
149
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
150
151
152
}

_EMBEDDING_MODELS = {
153
    # [Text-only]
154
    "BertModel": ("bert", "BertEmbeddingModel"),
155
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
156
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
157
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
158
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
159
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
160
    "GritLM": ("gritlm", "GritLM"),
161
162
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
163
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
164
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
165
    "LlamaModel": ("llama", "LlamaForCausalLM"),
166
167
168
169
170
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
171
    "MistralModel": ("llama", "LlamaForCausalLM"),
172
    "ModernBertModel": ("modernbert", "ModernBertModel"),
173
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
174
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
175
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
176
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
177
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
178
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
179
180
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
181
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
182
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
183
    # [Multimodal]
Cyrus Leung's avatar
Cyrus Leung committed
184
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
185
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
186
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
187
188
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
189
    # models for the time being.
190
191
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
192
193
}

194
195
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
196
197
198
199
    "GteNewForSequenceClassification": ("bert_with_rope",
                                        "GteNewForSequenceClassification"),
    "ModernBertForSequenceClassification": ("modernbert",
                                            "ModernBertForSequenceClassification"),
200
201
202
203
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
204
    # [Auto-converted (see adapters.py)]
205
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"), # noqa: E501,
206
207
}

208
_MULTIMODAL_MODELS = {
209
    # [Decoder-only]
210
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
Jennifer Zhao's avatar
Jennifer Zhao committed
211
    "AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"),  # noqa: E501
212
213
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
214
    "Cohere2VisionForConditionalGeneration": ("cohere2_vision", "Cohere2VisionForConditionalGeneration"),  # noqa: E501
215
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
216
    "Ernie4_5_VLMoeForConditionalGeneration": ("ernie45_vl", "Ernie4_5_VLMoeForConditionalGeneration"),  # noqa: E501
217
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
218
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
219
    "Gemma3nForConditionalGeneration": ("gemma3n_mm", "Gemma3nForConditionalGeneration"),    # noqa: E501
220
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
221
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),  # noqa: E501
Jee Jee Li's avatar
Jee Jee Li committed
222
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),  # noqa: E501
223
    "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"),  # noqa: E501
224
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
225
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
226
    "NemotronH_Nano_VL": ("nano_nemotron_vl", "NemotronH_Nano_VL"),
Lyu Han's avatar
Lyu Han committed
227
    "InternS1ForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
228
    "InternVLForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
229
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
230
    "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"),  # noqa: E501
231
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
232
    "KeyeVL1_5ForConditionalGeneration": ("keye_vl1_5", "KeyeVL1_5ForConditionalGeneration"), # noqa: E501
233
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
234
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
235
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
236
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
237
238
239
240
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
241
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
242
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
243
    "MiniMaxVL01ForConditionalGeneration": ("minimax_vl_01", "MiniMaxVL01ForConditionalGeneration"),  # noqa: E501
244
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
245
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
246
    "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"),  # noqa: E501
247
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
248
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
249
    "Ovis": ("ovis", "Ovis"),
250
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
251
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
252
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
253
254
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
    "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"),  # noqa: E501
255
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
256
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
257
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
258
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
259
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
260
    "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
261
    "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
262
    "UltravoxModel": ("ultravox", "UltravoxModel"),
263
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
264
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
265
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
266
    "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"),  # noqa: E501
Patrick von Platen's avatar
Patrick von Platen committed
267
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
268
    # [Encoder-decoder]
269
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
270
}
271
272

_SPECULATIVE_DECODING_MODELS = {
273
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
274
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
275
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
276
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
277
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
278
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
279
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
280
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
281
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
282
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
283
    "MedusaModel": ("medusa", "Medusa"),
284
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
285
286
287
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
288
}
289

290
_TRANSFORMERS_SUPPORTED_MODELS = {
291
292
293
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
294
295
296
297
    "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
}

_TRANSFORMERS_BACKEND_MODELS = {
298
    "TransformersModel": ("transformers", "TransformersModel"),
299
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
300
    "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
301
}
302
# yapf: enable
303

304
_VLLM_MODELS = {
305
    **_TEXT_GENERATION_MODELS,
306
    **_EMBEDDING_MODELS,
307
    **_CROSS_ENCODER_MODELS,
308
    **_MULTIMODAL_MODELS,
309
    **_SPECULATIVE_DECODING_MODELS,
310
311
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
312
313
}

314
315
316
317
318
319
320
321
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
_SUBPROCESS_COMMAND = [
    sys.executable, "-m", "vllm.model_executor.models.registry"
]

322
323
324
325
326
327
328
329
330
331
332
_PREVIOUSLY_SUPPORTED_MODELS = {
    "Phi3SmallForCausalLM": "0.9.2",
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "BartModel": "0.10.2",
    "BartForConditionalGeneration": "0.10.2",
    "DonutForConditionalGeneration": "0.10.2",
    "Florence2ForConditionalGeneration": "0.10.2",
    "MBartForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
333

334

335
336
@dataclass(frozen=True)
class _ModelInfo:
337
    architecture: str
338
    is_text_generation_model: bool
339
    is_pooling_model: bool
340
    default_pooling_type: str
341
    supports_cross_encoding: bool
342
    supports_multimodal: bool
343
    supports_multimodal_raw_input_only: bool
344
    supports_multimodal_encoder_tp_data: bool
345
    supports_pp: bool
346
347
    has_inner_state: bool
    is_attention_free: bool
348
    is_hybrid: bool
349
    has_noops: bool
350
    supports_transcription: bool
351
    supports_transcription_only: bool
352
    supports_v0_only: bool
353
354

    @staticmethod
355
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
356
        return _ModelInfo(
357
            architecture=model.__name__,
358
            is_text_generation_model=is_text_generation_model(model),
359
            is_pooling_model=is_pooling_model(model),
360
            default_pooling_type=get_default_pooling_type(model),
361
            supports_cross_encoding=supports_cross_encoding(model),
362
            supports_multimodal=supports_multimodal(model),
363
364
            supports_multimodal_raw_input_only=
            supports_multimodal_raw_input_only(model),
365
366
            supports_multimodal_encoder_tp_data=
            supports_multimodal_encoder_tp_data(model),
367
            supports_pp=supports_pp(model),
368
369
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
370
            is_hybrid=is_hybrid(model),
371
            supports_transcription=supports_transcription(model),
372
373
            supports_transcription_only=(supports_transcription(model) and
                                         model.supports_transcription_only),
374
            supports_v0_only=supports_v0_only(model),
375
            has_noops=has_noops(model),
376
        )
377
378


379
class _BaseRegisteredModel(ABC):
380

381
382
383
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
384

385
    @abstractmethod
386
    def load_model_cls(self) -> type[nn.Module]:
387
        raise NotImplementedError
388
389


390
391
392
393
394
395
396
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
397
    model_cls: type[nn.Module]
398
399

    @staticmethod
400
    def from_model_cls(model_cls: type[nn.Module]):
401
402
403
404
405
406
407
408
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

409
    def load_model_cls(self) -> type[nn.Module]:
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
    module_name: str
    class_name: str

    # Performed in another process to avoid initializing CUDA
    def inspect_model_cls(self) -> _ModelInfo:
        return _run_in_subprocess(
            lambda: _ModelInfo.from_model_cls(self.load_model_cls()))

426
    def load_model_cls(self) -> type[nn.Module]:
427
428
429
430
431
432
433
434
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
435
) -> Optional[type[nn.Module]]:
436
    from vllm.platforms import current_platform
437
    current_platform.verify_model_arch(model_arch)
438
439
440
441
442
443
    try:
        return model.load_model_cls()
    except Exception:
        logger.exception("Error in loading model architecture '%s'",
                         model_arch)
        return None
444
445


446
447
448
449
450
451
452
453
454
455
456
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
        logger.exception("Error in inspecting model architecture '%s'",
                         model_arch)
        return None
457
458


459
460
461
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
462
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
463

464
    def get_supported_archs(self) -> Set[str]:
465
        return self.models.keys()
466

467
468
469
    def register_model(
        self,
        model_arch: str,
470
        model_cls: Union[type[nn.Module], str],
471
    ) -> None:
472
473
474
        """
        Register an external model to be used in vLLM.

475
        `model_cls` can be either:
476

477
        - A [`torch.nn.Module`][] class directly referencing the model.
478
        - A string in the format `<module>:<class>` which can be used to
479
480
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
481
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
482
        """
483
484
485
486
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

487
        if model_arch in self.models:
488
489
490
            logger.warning(
                "Model architecture %s is already registered, and will be "
                "overwritten by the new model class %s.", model_arch,
491
492
493
494
495
496
497
                model_cls)

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
498

499
            model = _LazyRegisteredModel(*split_str)
500
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
501
            model = _RegisteredModel.from_model_cls(model_cls)
502
503
504
505
        else:
            msg = ("`model_cls` should be a string or PyTorch model class, "
                   f"not a {type(model_arch)}")
            raise TypeError(msg)
506

507
        self.models[model_arch] = model
508

509
    def _raise_for_unsupported(self, architectures: list[str]):
510
        all_supported_archs = self.get_supported_archs()
511

512
513
514
515
516
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
                "to be inspected. Please check the logs for more details.")

517
518
519
520
521
522
523
524
525
526
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
                    "use this model architecture.")

527
528
529
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
            f"Supported architectures: {all_supported_archs}")
530

531
    def _try_load_model_cls(self,
532
                            model_arch: str) -> Optional[type[nn.Module]]:
533
534
        if model_arch not in self.models:
            return None
535

536
        return _try_load_model_cls(model_arch, self.models[model_arch])
537

538
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
539
540
        if model_arch not in self.models:
            return None
541

542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> Optional[str]:
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

        auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map",
                                           None) or dict()

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
                if model_config.model_impl != ModelImpl.TRANSFORMERS:
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
                    "'auto_map' (relevant if the model is custom).")

        if not model_module.is_backend_compatible():
            if model_config.model_impl != ModelImpl.TRANSFORMERS:
599
                return None
600

601
602
603
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
                "is not compatible with vLLM.")
604

605
        return model_config._get_transformers_backend_cls()
606

607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
632

633
634
    def inspect_model_cls(
        self,
635
        architectures: Union[str, list[str]],
636
        model_config: ModelConfig,
637
    ) -> tuple[_ModelInfo, str]:
638
639
        if isinstance(architectures, str):
            architectures = [architectures]
640
641
        if not architectures:
            raise ValueError("No model architectures are specified")
642
643
644
645
646
647
648
649
650

        # Require transformers impl
        if model_config.model_impl == ModelImpl.TRANSFORMERS:
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
651
652
653
        elif model_config.model_impl == ModelImpl.TERRATORCH:
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
654

655
656
657
658
659
660
661
662
663
664
665
666
667
        # Fallback to transformers impl (after resolving convert_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO
                and getattr(model_config, "convert_type", "none") == "none"):
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
668
            model_info = self._try_inspect_model_cls(normalized_arch)
669
            if model_info is not None:
670
                return (model_info, arch)
671

672
673
674
        # Fallback to transformers impl (before resolving runner_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO):
675
676
677
678
679
680
681
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

682
        return self._raise_for_unsupported(architectures)
683

684
685
    def resolve_model_cls(
        self,
686
        architectures: Union[str, list[str]],
687
        model_config: ModelConfig,
688
    ) -> tuple[type[nn.Module], str]:
689
690
        if isinstance(architectures, str):
            architectures = [architectures]
691
692
        if not architectures:
            raise ValueError("No model architectures are specified")
693
694
695
696
697
698
699
700
701

        # Require transformers impl
        if model_config.model_impl == ModelImpl.TRANSFORMERS:
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
702
703
704
705
706
        elif model_config.model_impl == ModelImpl.TERRATORCH:
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
707

708
709
710
711
712
713
714
715
716
717
718
719
720
        # Fallback to transformers impl (after resolving convert_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO
                and getattr(model_config, "convert_type", "none") == "none"):
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
721
            model_cls = self._try_load_model_cls(normalized_arch)
722
723
            if model_cls is not None:
                return (model_cls, arch)
724

725
726
727
        # Fallback to transformers impl (before resolving runner_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO):
728
729
730
731
732
733
734
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

735
        return self._raise_for_unsupported(architectures)
736

737
738
    def is_text_generation_model(
        self,
739
        architectures: Union[str, list[str]],
740
        model_config: ModelConfig,
741
    ) -> bool:
742
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
743
        return model_cls.is_text_generation_model
744

745
    def is_pooling_model(
746
        self,
747
        architectures: Union[str, list[str]],
748
        model_config: ModelConfig,
749
    ) -> bool:
750
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
751
        return model_cls.is_pooling_model
752

753
754
    def is_cross_encoder_model(
        self,
755
        architectures: Union[str, list[str]],
756
        model_config: ModelConfig,
757
    ) -> bool:
758
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
759
        return model_cls.supports_cross_encoding
760

761
762
    def is_multimodal_model(
        self,
763
        architectures: Union[str, list[str]],
764
        model_config: ModelConfig,
765
    ) -> bool:
766
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
767
        return model_cls.supports_multimodal
768

769
    def is_multimodal_raw_input_only_model(
770
771
        self,
        architectures: Union[str, list[str]],
772
        model_config: ModelConfig,
773
    ) -> bool:
774
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
775
        return model_cls.supports_multimodal_raw_input_only
776

777
778
    def is_pp_supported_model(
        self,
779
        architectures: Union[str, list[str]],
780
        model_config: ModelConfig,
781
    ) -> bool:
782
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
783
        return model_cls.supports_pp
784

785
786
    def model_has_inner_state(
        self,
787
        architectures: Union[str, list[str]],
788
        model_config: ModelConfig,
789
    ) -> bool:
790
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
791
        return model_cls.has_inner_state
792

793
794
    def is_attention_free_model(
        self,
795
        architectures: Union[str, list[str]],
796
        model_config: ModelConfig,
797
    ) -> bool:
798
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
799
        return model_cls.is_attention_free
800

801
802
    def is_hybrid_model(
        self,
803
        architectures: Union[str, list[str]],
804
        model_config: ModelConfig,
805
    ) -> bool:
806
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
807
808
        return model_cls.is_hybrid

809
810
    def is_noops_model(
        self,
811
        architectures: Union[str, list[str]],
812
        model_config: ModelConfig,
813
    ) -> bool:
814
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
815
816
        return model_cls.has_noops

817
818
    def is_transcription_model(
        self,
819
        architectures: Union[str, list[str]],
820
        model_config: ModelConfig,
821
    ) -> bool:
822
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
823
824
        return model_cls.supports_transcription

825
826
827
    def is_transcription_only_model(
        self,
        architectures: Union[str, list[str]],
828
        model_config: ModelConfig,
829
    ) -> bool:
830
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
831
832
        return model_cls.supports_transcription_only

833
834
    def is_v1_compatible(
        self,
835
        architectures: Union[str, list[str]],
836
        model_config: ModelConfig,
837
    ) -> bool:
838
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
839
840
        return not model_cls.supports_v0_only

841
842

ModelRegistry = _ModelRegistry({
843
844
    model_arch:
    _LazyRegisteredModel(
845
846
847
848
849
850
851
852
853
854
        module_name=f"vllm.model_executor.models.{mod_relname}",
        class_name=cls_name,
    )
    for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
})

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
855
856
857
858
859
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

860
        # `cloudpickle` allows pickling lambda functions directly
861
        import cloudpickle
862
        input_bytes = cloudpickle.dumps((fn, output_filepath))
863
864
865

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
866
867
868
        returned = subprocess.run(_SUBPROCESS_COMMAND,
                                  input=input_bytes,
                                  capture_output=True)
869
870
871
872
873
874
875
876
877

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
            raise RuntimeError(f"Error raised in subprocess:\n"
                               f"{returned.stderr.decode()}") from e

878
        with open(output_filepath, "rb") as f:
879
880
881
882
883
884
885
886
887
888
889
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
890
891
892

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
893
894
895


if __name__ == "__main__":
896
    _run()