registry.py 36.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7
import importlib
8
import os
9
import pickle
10
11
import subprocess
import sys
12
import tempfile
13
from abc import ABC, abstractmethod
14
from collections.abc import Set
15
from dataclasses import dataclass, field
16
from functools import lru_cache
17
from typing import Callable, Optional, TypeVar, Union
18
19

import torch.nn as nn
20
import transformers
21

22
23
from vllm.config import (ModelConfig, ModelImpl, iter_architecture_defaults,
                         try_match_architecture_defaults)
24
from vllm.logger import init_logger
25
26
from vllm.transformers_utils.dynamic_module import (
    try_get_class_from_dynamic_module)
27

28
29
from .interfaces import (get_default_pooling_type, has_inner_state, has_noops,
                         is_attention_free, is_hybrid, supports_cross_encoding,
30
31
                         supports_multimodal, supports_multimodal_raw_input,
                         supports_pp, supports_transcription, supports_v0_only)
32
from .interfaces_base import is_pooling_model, is_text_generation_model
33
34
35

logger = init_logger(__name__)

36
# yapf: disable
37
38
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
39
40
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
41
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
42
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
43
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
44
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
45
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
46
47
48
49
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
50
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
51
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
52
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
53
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
54
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
55
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
56
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
57
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
58
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
59
60
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
61
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
62
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
63
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
64
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
65
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
66
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
67
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
68
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
69
70
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
71
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
72
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
73
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
74
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
75
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
76
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
77
78
79
80
81
82
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
83
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),   # noqa: E501
84
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),   # noqa: E501
85
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
86
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
87
88
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
89
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
90
91
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
92
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
93
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
94
95
96
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
97
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),  # noqa: E501
98
99
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
100
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
101
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
Dhia Eddine Rhaiem's avatar
Dhia Eddine Rhaiem committed
102
    "FalconH1ForCausalLM":("falcon_h1", "FalconH1ForCausalLM"),
103
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
104
105
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
106
107
108
109
110
111
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
112
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
113
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
114
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
115
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
116
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
117
118
119
120
121
122
123
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
124
    "Phi4FlashForCausalLM": ("phi4flash", "Phi4FlashForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
125
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
126
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
127
128
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
129
130
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
131
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
Song's avatar
Song committed
132
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
133
134
135
136
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
137
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
138
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
139
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
140
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
141
142
143
    # [Encoder-decoder]
    "BartModel": ("bart", "BartForConditionalGeneration"),
    "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
汪志鹏's avatar
汪志鹏 committed
144
    "MBartForConditionalGeneration": ("bart", "MBartForConditionalGeneration"),
145
146
147
}

_EMBEDDING_MODELS = {
148
    # [Text-only]
149
    "BertModel": ("bert", "BertEmbeddingModel"),
150
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
151
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
152
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
153
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
154
    "GritLM": ("gritlm", "GritLM"),
155
156
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
157
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
158
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
159
    "LlamaModel": ("llama", "LlamaForCausalLM"),
160
161
162
163
164
    **{
        # Multiple models share the same architecture, so we include them all
        k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
        if arch == "LlamaForCausalLM"
    },
165
    "MistralModel": ("llama", "LlamaForCausalLM"),
166
    "ModernBertModel": ("modernbert", "ModernBertModel"),
167
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
168
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
169
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
170
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
171
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
172
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
173
174
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
175
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
176
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
177
    # [Multimodal]
Cyrus Leung's avatar
Cyrus Leung committed
178
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
179
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
180
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
181
182
183
184
    # Technically PrithviGeoSpatialMAE is a model that works on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
    # models for the time being.
    "PrithviGeoSpatialMAE": ("prithvi_geospatial_mae", "PrithviGeoSpatialMAE"),
185
186
}

187
188
189
190
191
192
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "RobertaForSequenceClassification": ("roberta",
                                         "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": ("roberta",
                                            "RobertaForSequenceClassification"),
xsank's avatar
xsank committed
193
194
    "ModernBertForSequenceClassification": ("modernbert",
                                            "ModernBertForSequenceClassification"),
195
    # [Auto-converted (see adapters.py)]
196
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"), # noqa: E501,
197
198
}

199
_MULTIMODAL_MODELS = {
200
    # [Decoder-only]
201
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
Jennifer Zhao's avatar
Jennifer Zhao committed
202
    "AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"),  # noqa: E501
203
204
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
    "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"),  # noqa: E501
205
    "Cohere2VisionForConditionalGeneration": ("cohere2_vision", "Cohere2VisionForConditionalGeneration"),  # noqa: E501
206
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
207
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
208
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
209
    "Gemma3nForConditionalGeneration": ("gemma3n_mm", "Gemma3nForConditionalGeneration"),    # noqa: E501
210
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
211
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),  # noqa: E501
Jee Jee Li's avatar
Jee Jee Li committed
212
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),  # noqa: E501
213
    "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"),  # noqa: E501
214
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
215
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
Lyu Han's avatar
Lyu Han committed
216
    "InternS1ForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"),  # noqa: E501
217
    "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
218
    "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"),  # noqa: E501
219
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
220
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
221
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
222
223
224
225
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
    "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
    "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),  # noqa: E501
    "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),  # noqa: E501
226
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
227
    "MiniMaxVL01ForConditionalGeneration": ("minimax_vl_01", "MiniMaxVL01ForConditionalGeneration"),  # noqa: E501
228
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
229
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
230
    "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"),  # noqa: E501
231
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
232
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
233
    "Ovis": ("ovis", "Ovis"),
234
    "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"),  # noqa: E501
235
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
236
237
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
    "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"),  # noqa: E501
238
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
239
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
240
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
241
    "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),  # noqa: E501
242
    "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),  # noqa: E501
243
    "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
244
    "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"),  # noqa: E501
245
    "UltravoxModel": ("ultravox", "UltravoxModel"),
Song's avatar
Song committed
246
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
247
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
248
    "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"),  # noqa: E501
Patrick von Platen's avatar
Patrick von Platen committed
249
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
250
    # [Encoder-decoder]
251
    "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"),  # noqa: E501
252
    "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"),  # noqa: E501
253
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
254
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
255
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
256
}
257
258

_SPECULATIVE_DECODING_MODELS = {
259
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
260
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
261
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
262
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
263
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
264
265
    # TODO: Re-enable this once tests/models/test_initialization.py is fixed, see PR #22333 #22611  # noqa: E501
    # "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
266
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
267
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
268
    "MedusaModel": ("medusa", "Medusa"),
269
270
271
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
272
}
273

274
_TRANSFORMERS_SUPPORTED_MODELS = {
275
276
277
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
278
279
280
281
    "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
}

_TRANSFORMERS_BACKEND_MODELS = {
282
    "TransformersModel": ("transformers", "TransformersModel"),
283
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
284
    "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
285
}
286
# yapf: enable
287

288
_VLLM_MODELS = {
289
    **_TEXT_GENERATION_MODELS,
290
    **_EMBEDDING_MODELS,
291
    **_CROSS_ENCODER_MODELS,
292
    **_MULTIMODAL_MODELS,
293
    **_SPECULATIVE_DECODING_MODELS,
294
295
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
296
297
}

298
299
300
301
302
303
304
305
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
_SUBPROCESS_COMMAND = [
    sys.executable, "-m", "vllm.model_executor.models.registry"
]

306
307
_PREVIOUSLY_SUPPORTED_MODELS = {"Phi3SmallForCausalLM": "0.9.2"}

308

309
310
@dataclass(frozen=True)
class _ModelInfo:
311
    architecture: str
312
    is_text_generation_model: bool
313
    is_pooling_model: bool
314
    default_pooling_type: str
315
    supports_cross_encoding: bool
316
    supports_multimodal: bool
317
    supports_multimodal_raw_input: bool
318
    supports_pp: bool
319
320
    has_inner_state: bool
    is_attention_free: bool
321
    is_hybrid: bool
322
    has_noops: bool
323
    supports_transcription: bool
324
    supports_transcription_only: bool
325
    supports_v0_only: bool
326
327

    @staticmethod
328
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
329
        return _ModelInfo(
330
            architecture=model.__name__,
331
            is_text_generation_model=is_text_generation_model(model),
332
            is_pooling_model=is_pooling_model(model),
333
            default_pooling_type=get_default_pooling_type(model),
334
            supports_cross_encoding=supports_cross_encoding(model),
335
            supports_multimodal=supports_multimodal(model),
336
            supports_multimodal_raw_input=supports_multimodal_raw_input(model),
337
            supports_pp=supports_pp(model),
338
339
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
340
            is_hybrid=is_hybrid(model),
341
            supports_transcription=supports_transcription(model),
342
343
            supports_transcription_only=(supports_transcription(model) and
                                         model.supports_transcription_only),
344
            supports_v0_only=supports_v0_only(model),
345
            has_noops=has_noops(model),
346
        )
347
348


349
class _BaseRegisteredModel(ABC):
350

351
352
353
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
354

355
    @abstractmethod
356
    def load_model_cls(self) -> type[nn.Module]:
357
        raise NotImplementedError
358
359


360
361
362
363
364
365
366
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
367
    model_cls: type[nn.Module]
368
369

    @staticmethod
370
    def from_model_cls(model_cls: type[nn.Module]):
371
372
373
374
375
376
377
378
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

379
    def load_model_cls(self) -> type[nn.Module]:
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
    module_name: str
    class_name: str

    # Performed in another process to avoid initializing CUDA
    def inspect_model_cls(self) -> _ModelInfo:
        return _run_in_subprocess(
            lambda: _ModelInfo.from_model_cls(self.load_model_cls()))

396
    def load_model_cls(self) -> type[nn.Module]:
397
398
399
400
401
402
403
404
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
405
) -> Optional[type[nn.Module]]:
406
    from vllm.platforms import current_platform
407
    current_platform.verify_model_arch(model_arch)
408
409
410
411
412
413
    try:
        return model.load_model_cls()
    except Exception:
        logger.exception("Error in loading model architecture '%s'",
                         model_arch)
        return None
414
415


416
417
418
419
420
421
422
423
424
425
426
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
        logger.exception("Error in inspecting model architecture '%s'",
                         model_arch)
        return None
427
428


429
430
431
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
432
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
433

434
    def get_supported_archs(self) -> Set[str]:
435
        return self.models.keys()
436

437
438
439
    def register_model(
        self,
        model_arch: str,
440
        model_cls: Union[type[nn.Module], str],
441
    ) -> None:
442
443
444
        """
        Register an external model to be used in vLLM.

445
        `model_cls` can be either:
446

447
        - A [`torch.nn.Module`][] class directly referencing the model.
448
        - A string in the format `<module>:<class>` which can be used to
449
450
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
451
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
452
        """
453
454
455
456
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

457
        if model_arch in self.models:
458
459
460
            logger.warning(
                "Model architecture %s is already registered, and will be "
                "overwritten by the new model class %s.", model_arch,
461
462
463
464
465
466
467
                model_cls)

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
468

469
            model = _LazyRegisteredModel(*split_str)
470
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
471
            model = _RegisteredModel.from_model_cls(model_cls)
472
473
474
475
        else:
            msg = ("`model_cls` should be a string or PyTorch model class, "
                   f"not a {type(model_arch)}")
            raise TypeError(msg)
476

477
        self.models[model_arch] = model
478

479
    def _raise_for_unsupported(self, architectures: list[str]):
480
        all_supported_archs = self.get_supported_archs()
481

482
483
484
485
486
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
                "to be inspected. Please check the logs for more details.")

487
488
489
490
491
492
493
494
495
496
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
                    "use this model architecture.")

497
498
499
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
            f"Supported architectures: {all_supported_archs}")
500

501
    def _try_load_model_cls(self,
502
                            model_arch: str) -> Optional[type[nn.Module]]:
503
504
        if model_arch not in self.models:
            return None
505

506
        return _try_load_model_cls(model_arch, self.models[model_arch])
507

508
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
509
510
        if model_arch not in self.models:
            return None
511

512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> Optional[str]:
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

        auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map",
                                           None) or dict()

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
                if model_config.model_impl != ModelImpl.TRANSFORMERS:
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
                    "'auto_map' (relevant if the model is custom).")

        if not model_module.is_backend_compatible():
            if model_config.model_impl != ModelImpl.TRANSFORMERS:
569
                return None
570

571
572
573
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
                "is not compatible with vLLM.")
574

575
        return model_config._get_transformers_backend_cls()
576

577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
602

603
604
    def inspect_model_cls(
        self,
605
        architectures: Union[str, list[str]],
606
        model_config: ModelConfig,
607
    ) -> tuple[_ModelInfo, str]:
608
609
        if isinstance(architectures, str):
            architectures = [architectures]
610
611
        if not architectures:
            raise ValueError("No model architectures are specified")
612
613
614
615
616
617
618
619
620
621

        # Require transformers impl
        if model_config.model_impl == ModelImpl.TRANSFORMERS:
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

622
623
624
625
626
627
628
629
630
631
632
633
634
        # Fallback to transformers impl (after resolving convert_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO
                and getattr(model_config, "convert_type", "none") == "none"):
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
635
            model_info = self._try_inspect_model_cls(normalized_arch)
636
            if model_info is not None:
637
                return (model_info, arch)
638

639
640
641
        # Fallback to transformers impl (before resolving runner_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO):
642
643
644
645
646
647
648
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

649
        return self._raise_for_unsupported(architectures)
650

651
652
    def resolve_model_cls(
        self,
653
        architectures: Union[str, list[str]],
654
        model_config: ModelConfig,
655
    ) -> tuple[type[nn.Module], str]:
656
657
        if isinstance(architectures, str):
            architectures = [architectures]
658
659
        if not architectures:
            raise ValueError("No model architectures are specified")
660
661
662
663
664
665
666
667
668
669

        # Require transformers impl
        if model_config.model_impl == ModelImpl.TRANSFORMERS:
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

670
671
672
673
674
675
676
677
678
679
680
681
682
        # Fallback to transformers impl (after resolving convert_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO
                and getattr(model_config, "convert_type", "none") == "none"):
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
683
            model_cls = self._try_load_model_cls(normalized_arch)
684
685
            if model_cls is not None:
                return (model_cls, arch)
686

687
688
689
        # Fallback to transformers impl (before resolving runner_type)
        if (all(arch not in self.models for arch in architectures)
                and model_config.model_impl == ModelImpl.AUTO):
690
691
692
693
694
695
696
            arch = self._try_resolve_transformers(architectures[0],
                                                  model_config)
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

697
        return self._raise_for_unsupported(architectures)
698

699
700
    def is_text_generation_model(
        self,
701
        architectures: Union[str, list[str]],
702
        model_config: ModelConfig,
703
    ) -> bool:
704
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
705
        return model_cls.is_text_generation_model
706

707
    def is_pooling_model(
708
        self,
709
        architectures: Union[str, list[str]],
710
        model_config: ModelConfig,
711
    ) -> bool:
712
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
713
        return model_cls.is_pooling_model
714

715
716
    def is_cross_encoder_model(
        self,
717
        architectures: Union[str, list[str]],
718
        model_config: ModelConfig,
719
    ) -> bool:
720
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
721
        return model_cls.supports_cross_encoding
722

723
724
    def is_multimodal_model(
        self,
725
        architectures: Union[str, list[str]],
726
        model_config: ModelConfig,
727
    ) -> bool:
728
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
729
        return model_cls.supports_multimodal
730

731
732
733
    def supports_multimodal_raw_input(
        self,
        architectures: Union[str, list[str]],
734
        model_config: ModelConfig,
735
    ) -> bool:
736
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
737
738
        return model_cls.supports_multimodal_raw_input

739
740
    def is_pp_supported_model(
        self,
741
        architectures: Union[str, list[str]],
742
        model_config: ModelConfig,
743
    ) -> bool:
744
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
745
        return model_cls.supports_pp
746

747
748
    def model_has_inner_state(
        self,
749
        architectures: Union[str, list[str]],
750
        model_config: ModelConfig,
751
    ) -> bool:
752
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
753
        return model_cls.has_inner_state
754

755
756
    def is_attention_free_model(
        self,
757
        architectures: Union[str, list[str]],
758
        model_config: ModelConfig,
759
    ) -> bool:
760
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
761
        return model_cls.is_attention_free
762

763
764
    def is_hybrid_model(
        self,
765
        architectures: Union[str, list[str]],
766
        model_config: ModelConfig,
767
    ) -> bool:
768
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
769
770
        return model_cls.is_hybrid

771
772
    def is_noops_model(
        self,
773
        architectures: Union[str, list[str]],
774
        model_config: ModelConfig,
775
    ) -> bool:
776
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
777
778
        return model_cls.has_noops

779
780
    def is_transcription_model(
        self,
781
        architectures: Union[str, list[str]],
782
        model_config: ModelConfig,
783
    ) -> bool:
784
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
785
786
        return model_cls.supports_transcription

787
788
789
    def is_transcription_only_model(
        self,
        architectures: Union[str, list[str]],
790
        model_config: ModelConfig,
791
    ) -> bool:
792
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
793
794
        return model_cls.supports_transcription_only

795
796
    def is_v1_compatible(
        self,
797
        architectures: Union[str, list[str]],
798
        model_config: ModelConfig,
799
    ) -> bool:
800
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
801
802
        return not model_cls.supports_v0_only

803
804

ModelRegistry = _ModelRegistry({
805
806
    model_arch:
    _LazyRegisteredModel(
807
808
809
810
811
812
813
814
815
816
        module_name=f"vllm.model_executor.models.{mod_relname}",
        class_name=cls_name,
    )
    for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
})

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
817
818
819
820
821
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

822
        # `cloudpickle` allows pickling lambda functions directly
823
        import cloudpickle
824
        input_bytes = cloudpickle.dumps((fn, output_filepath))
825
826
827

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
828
829
830
        returned = subprocess.run(_SUBPROCESS_COMMAND,
                                  input=input_bytes,
                                  capture_output=True)
831
832
833
834
835
836
837
838
839

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
            raise RuntimeError(f"Error raised in subprocess:\n"
                               f"{returned.stderr.decode()}") from e

840
        with open(output_filepath, "rb") as f:
841
842
843
844
845
846
847
848
849
850
851
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
852
853
854

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
855
856
857


if __name__ == "__main__":
858
    _run()