"vllm/entrypoints/openai/chat_completion/serving.py" did not exist on "5f2cacdb1e62e468c707eab168c6d6961254bc93"
registry.py 51.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import importlib
9
import json
10
import os
11
import pickle
12
13
import subprocess
import sys
14
import tempfile
15
from abc import ABC, abstractmethod
16
from collections.abc import Callable, Set
17
from dataclasses import asdict, dataclass, field
18
from functools import lru_cache
19
from pathlib import Path
20
from typing import TYPE_CHECKING, Any, TypeVar
21
22

import torch.nn as nn
23
import transformers
24

25
from vllm import envs
26
27
28
29
30
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
31
from vllm.logger import init_logger
32
from vllm.logging_utils import logtime
33
from vllm.tasks import ScoreType
34
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
35
from vllm.utils.hashing import safe_hash
36

37
38
if TYPE_CHECKING:
    from vllm.config.model import AttnTypeStr
39
    from vllm.config.pooler import SequencePoolingType, TokenPoolingType
40
41
else:
    AttnTypeStr = Any
42
43
    SequencePoolingType = Any
    TokenPoolingType = Any
44
45


46
47
48
49
50
from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
Patrick von Platen's avatar
Patrick von Platen committed
51
    requires_raw_input_tokens,
52
    supports_mamba_prefix_caching,
53
54
55
56
57
58
59
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
)
from .interfaces_base import (
60
    get_attn_type,
61
62
    get_default_seq_pooling_type,
    get_default_tok_pooling_type,
63
    get_score_type,
64
65
66
    is_pooling_model,
    is_text_generation_model,
)
67
68
69

logger = init_logger(__name__)

70
71
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
72
    "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
73
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
74
75
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
76
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
77
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
78
    "AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
79
80
81
82
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
83
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
84
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
85
    "BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
86
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
87
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
88
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
89
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
90
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
91
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
92
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
93
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
94
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
95
    "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
96
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
97
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
98
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
99
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
100
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
101
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
102
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
103
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
Kyungmin Lee's avatar
Kyungmin Lee committed
104
    "ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
105
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
106
107
108
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
109
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
110
111
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
112
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
113
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
114
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
115
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
116
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
117
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
118
    "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
Jee Jee Li's avatar
Jee Jee Li committed
119
    "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
120
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
121
122
123
124
125
126
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
127
128
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),  # noqa: E501
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),  # noqa: E501
129
    "GritLM": ("gritlm", "GritLM"),
Bijaya Dangol's avatar
Bijaya Dangol committed
130
131
    "Grok1ModelForCausalLM": ("grok1", "GrokForCausalLM"),
    "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
132
133
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
134
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
135
    "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
136
    "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
137
138
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
139
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
140
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
141
142
    "IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
    "IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
143
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
144
    "Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
145
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
146
    "KimiLinearForCausalLM": ("kimi_linear", "KimiLinearForCausalLM"),  # noqa: E501
147
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
148
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
149
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
150
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
151
152
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
153
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
154
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
155
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
156
157
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
158
159
160
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
161
    "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
162
    "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
163
    "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
164
165
166
167
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
168
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
169
    "MiMoV2FlashForCausalLM": ("mimo_v2_flash", "MiMoV2FlashForCausalLM"),
170
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
171
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
172
    "NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
173
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
174
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
175
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
176
    "OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
177
178
179
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
180
    "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
181
    "PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
182
    "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
183
    "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
184
185
186
187
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
188
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
189
    "Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
190
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
191
192
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
193
194
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
195
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
196
197
    "SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
    "SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
198
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Li Xie's avatar
Li Xie committed
199
    "Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
Song's avatar
Song committed
200
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
csy0225's avatar
csy0225 committed
201
    "Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
202
203
204
205
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
206
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
207
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
208
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
209
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
210
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
211
212
213
}

_EMBEDDING_MODELS = {
214
    # [Text-only]
215
    "BertModel": ("bert", "BertEmbeddingModel"),
216
    "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
217
    "ErnieModel": ("ernie", "ErnieEmbeddingModel"),
218
    "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
219
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
220
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
221
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
222
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
223
    "GritLM": ("gritlm", "GritLM"),
224
225
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
226
    "LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
227
    "LlamaModel": ("llama", "LlamaForCausalLM"),
228
229
    **{
        # Multiple models share the same architecture, so we include them all
230
231
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
232
233
        if arch == "LlamaForCausalLM"
    },
234
    "MistralModel": ("llama", "LlamaForCausalLM"),
235
    "ModernBertModel": ("modernbert", "ModernBertModel"),
236
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
237
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
238
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
239
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
240
241
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
242
    "TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
243
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
chengchengpei's avatar
chengchengpei committed
244
245
246
247
    "VoyageQwen3BidirectionalEmbedModel": (
        "voyage",
        "VoyageQwen3BidirectionalEmbedModel",
    ),
248
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
249
    # [Multimodal]
250
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
251
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
252
253
254
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
255
    ),
256
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
257
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
258
    "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
259
260
261
262
    "LlamaNemotronVLModel": (
        "nemotron_vl",
        "LlamaNemotronVLForEmbedding",
    ),
263
264
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
265
    # models for the time being.
266
267
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
268
269
}

270
271
272
273
274
275
276
_LATE_INTERACTION_MODELS = {
    # [Text-only]
    "HF_ColBERT": ("colbert", "ColBERTModel"),
    "ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
    "ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
    # [Multimodal]
    "ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
277
    "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
278
279
    "ColQwen3": ("colqwen3", "ColQwen3Model"),
    "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
280
    "ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
281
282
283
284
285
286
287
288
289
290
    "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
}

_REWARD_MODELS = {
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
}

_TOKEN_CLASSIFICATION_MODELS = {
291
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
292
    "ErnieForTokenClassification": ("ernie", "ErnieForTokenClassification"),
293
294
295
296
297
298
299
300
301
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
}

_SEQUENCE_CLASSIFICATION_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
302
    "ErnieForSequenceClassification": ("ernie", "ErnieForSequenceClassification"),
303
304
305
306
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
307
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
308
309
310
    "LlamaBidirectionalForSequenceClassification": (
        "llama",
        "LlamaBidirectionalForSequenceClassification",
311
    ),
312
313
314
315
316
317
318
319
320
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
321
322
323
324
325
326
    # [Multimodal]
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
    "LlamaNemotronVLForSequenceClassification": (
        "nemotron_vl",
        "LlamaNemotronVLForSequenceClassification",
    ),
327
328
}

329
_MULTIMODAL_MODELS = {
330
    # [Decoder-only]
331
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
332
333
334
335
    "AudioFlamingo3ForConditionalGeneration": (
        "audioflamingo3",
        "AudioFlamingo3ForConditionalGeneration",
    ),
336
337
338
339
    "MusicFlamingoForConditionalGeneration": (
        "musicflamingo",
        "MusicFlamingoForConditionalGeneration",
    ),
340
341
342
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
343
    ),
344
    "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
345
    "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
346
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
347
348
349
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
350
    ),
351
352
353
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
354
    ),
355
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
356
    "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
RED's avatar
RED committed
357
    "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
358
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
359
360
361
362
    "Eagle2_5_VLForConditionalGeneration": (
        "eagle2_5_vl",
        "Eagle2_5_VLForConditionalGeneration",
    ),
363
364
365
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
366
    ),
367
368
369
370
    "FireRedASR2ForConditionalGeneration": (
        "fireredasr2",
        "FireRedASR2ForConditionalGeneration",
    ),
371
    "FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),  # noqa: E501
372
373
374
375
    "FunAudioChatForConditionalGeneration": (
        "funaudiochat",
        "FunAudioChatForConditionalGeneration",
    ),
376
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
377
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
378
379
380
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
381
    ),
382
    "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
383
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
384
385
386
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
    "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),  # noqa: E501
387
388
389
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
390
    ),
391
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
392
393
394
395
    "HunYuanVLForConditionalGeneration": (
        "hunyuan_vision",
        "HunYuanVLForConditionalGeneration",
    ),
ltd0924's avatar
ltd0924 committed
396
    "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
397
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
398
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
Zero's avatar
Zero committed
399
400
401
402
    "OpenCUAForConditionalGeneration": (
        "opencua",
        "OpenCUAForConditionalGeneration",
    ),
403
404
405
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
406
    ),
407
408
409
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
410
    ),
zxy's avatar
zxy committed
411
412
413
414
    "InternS1ProForConditionalGeneration": (
        "interns1_pro",
        "InternS1ProForConditionalGeneration",
    ),
415
416
417
418
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
oscardev256's avatar
oscardev256 committed
419
    "IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
420
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),  # noqa: E501
421
    "KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
422
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
423
424
425
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
426
    ),
427
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
428
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
Roger Wang's avatar
Roger Wang committed
429
    "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),  # noqa: E501
430
    "MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),  # noqa: E501
431
432
433
434
    "LightOnOCRForConditionalGeneration": (
        "lightonocr",
        "LightOnOCRForConditionalGeneration",
    ),
435
    "Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
436
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
437
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
438
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
439
440
441
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
442
    ),
443
444
445
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
446
    ),
447
448
449
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
450
    ),
451
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
452
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
453
454
455
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
456
    ),
457
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
458
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
459
460
461
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
462
    ),
463
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
464
    "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
465
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
466
467
468
469
    "OpenPanguVLForConditionalGeneration": (
        "openpangu_vl",
        "OpenPanguVLForConditionalGeneration",
    ),
470
    "Ovis": ("ovis", "Ovis"),
471
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
472
473
    "Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
    "Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
474
475
476
477
    "PaddleOCRVLForConditionalGeneration": (
        "paddleocr_vl",
        "PaddleOCRVLForConditionalGeneration",
    ),
478
479
480
481
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
    ),
482
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
483
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
484
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
485
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
486
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
487
488
489
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
490
    ),
491
492
493
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
494
    ),
495
496
497
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
498
    ),
499
500
501
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
502
    ),
503
504
505
506
    "Qwen3OmniMoeForConditionalGeneration": (
        "qwen3_omni_moe_thinker",
        "Qwen3OmniMoeThinkerForConditionalGeneration",
    ),
Roger Wang's avatar
Roger Wang committed
507
508
509
510
    "Qwen3ASRForConditionalGeneration": (
        "qwen3_asr",
        "Qwen3ASRForConditionalGeneration",
    ),
511
512
513
514
    "Qwen3ASRRealtimeGeneration": (
        "qwen3_asr_realtime",
        "Qwen3ASRRealtimeGeneration",
    ),
515
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),  # noqa: E501
516
517
518
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
519
    ),
520
521
522
523
524
525
526
527
    "Qwen3_5ForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5ForConditionalGeneration",
    ),
    "Qwen3_5MoeForConditionalGeneration": (
        "qwen3_5",
        "Qwen3_5MoeForConditionalGeneration",
    ),
528
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
529
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
530
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
531
532
533
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
534
    ),
535
    "UltravoxModel": ("ultravox", "UltravoxModel"),
Patrick von Platen's avatar
Patrick von Platen committed
536
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
537
    "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),  # noqa: E501
538
    # [Encoder-decoder]
Ekagra Ranjan's avatar
Ekagra Ranjan committed
539
540
541
542
    "CohereASRForConditionalGeneration": (
        "cohere_asr",
        "CohereASRForConditionalGeneration",
    ),
543
544
545
546
    "NemotronParseForConditionalGeneration": (
        "nemotron_parse",
        "NemotronParseForConditionalGeneration",
    ),
547
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
548
}
549
550

_SPECULATIVE_DECODING_MODELS = {
551
    "ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
552
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
553
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
554
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
555
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
556
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
557
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
558
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
559
    "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
560
561
562
563
    "EagleMistralLarge3ForCausalLM": (
        "mistral_large_3_eagle",
        "EagleMistralLarge3ForCausalLM",
    ),
564
565
    "Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
    "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
566
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
567
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
568
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
Kyungmin Lee's avatar
Kyungmin Lee committed
569
    "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
570
    "NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
XuruiYang's avatar
XuruiYang committed
571
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
572
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
573
    "Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
574
    "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
575
    "MedusaModel": ("medusa", "Medusa"),
576
    "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
577
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
csy0225's avatar
csy0225 committed
578
    "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
579
580
    "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
    "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
581
582
583
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
584
}
585

586
_TRANSFORMERS_SUPPORTED_MODELS = {
587
588
589
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
590
591
592
593
    "Emu3ForConditionalGeneration": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
594
595
596
}

_TRANSFORMERS_BACKEND_MODELS = {
597
    # Text generation models
598
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
    "TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
    # Multimodal models
    "TransformersMultiModalForCausalLM": (
        "transformers",
        "TransformersMultiModalForCausalLM",
    ),
    "TransformersMultiModalMoEForCausalLM": (
        "transformers",
        "TransformersMultiModalMoEForCausalLM",
    ),
    # Embedding models
    "TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
    "TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
    "TransformersMultiModalEmbeddingModel": (
        "transformers",
        "TransformersMultiModalEmbeddingModel",
    ),
    # Sequence classification models
617
    "TransformersForSequenceClassification": (
618
        "transformers",
619
        "TransformersForSequenceClassification",
620
    ),
621
    "TransformersMoEForSequenceClassification": (
622
        "transformers",
623
        "TransformersMoEForSequenceClassification",
624
    ),
625
626
627
    "TransformersMultiModalForSequenceClassification": (
        "transformers",
        "TransformersMultiModalForSequenceClassification",
628
    ),
629
}
630

631
_VLLM_MODELS = {
632
    **_TEXT_GENERATION_MODELS,
633
    **_EMBEDDING_MODELS,
634
635
636
637
    **_LATE_INTERACTION_MODELS,
    **_REWARD_MODELS,
    **_TOKEN_CLASSIFICATION_MODELS,
    **_SEQUENCE_CLASSIFICATION_MODELS,
638
    **_MULTIMODAL_MODELS,
639
    **_SPECULATIVE_DECODING_MODELS,
640
641
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
642
643
}

644
645
646
647
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
648
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
649

650
_PREVIOUSLY_SUPPORTED_MODELS = {
651
    "MotifForCausalLM": "0.10.2",
652
    "Phi3SmallForCausalLM": "0.9.2",
653
    "Phi4FlashForCausalLM": "0.10.2",
654
    "Phi4MultimodalForCausalLM": "0.12.0",
655
656
657
658
659
660
661
662
663
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "BartModel": "0.10.2",
    "BartForConditionalGeneration": "0.10.2",
    "DonutForConditionalGeneration": "0.10.2",
    "Florence2ForConditionalGeneration": "0.10.2",
    "MBartForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
664

665

666
667
@dataclass(frozen=True)
class _ModelInfo:
668
    architecture: str
669
    is_text_generation_model: bool
670
    is_pooling_model: bool
671
    attn_type: AttnTypeStr
672
673
    default_seq_pooling_type: SequencePoolingType
    default_tok_pooling_type: TokenPoolingType
674
    score_type: ScoreType
675
    supports_multimodal: bool
676
    supports_multimodal_raw_input_only: bool
Patrick von Platen's avatar
Patrick von Platen committed
677
    requires_raw_input_tokens: bool
678
    supports_multimodal_encoder_tp_data: bool
679
    supports_pp: bool
680
681
    has_inner_state: bool
    is_attention_free: bool
682
    is_hybrid: bool
683
    has_noops: bool
684
    supports_mamba_prefix_caching: bool
685
    supports_transcription: bool
686
    supports_transcription_only: bool
687
688

    @staticmethod
689
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
690
        return _ModelInfo(
691
            architecture=model.__name__,
692
            is_text_generation_model=is_text_generation_model(model),
693
            is_pooling_model=is_pooling_model(model),
694
695
            default_seq_pooling_type=get_default_seq_pooling_type(model),
            default_tok_pooling_type=get_default_tok_pooling_type(model),
696
            attn_type=get_attn_type(model),
697
            score_type=get_score_type(model),
698
            supports_multimodal=supports_multimodal(model),
699
700
701
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
Patrick von Platen's avatar
Patrick von Platen committed
702
            requires_raw_input_tokens=requires_raw_input_tokens(model),
703
704
705
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
706
            supports_pp=supports_pp(model),
707
708
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
709
            is_hybrid=is_hybrid(model),
710
            supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
711
            supports_transcription=supports_transcription(model),
712
713
714
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
715
            has_noops=has_noops(model),
716
        )
717
718


719
720
721
722
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
723

724
    @abstractmethod
725
    def load_model_cls(self) -> type[nn.Module]:
726
        raise NotImplementedError
727
728


729
730
731
732
733
734
735
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
736
    model_cls: type[nn.Module]
737
738

    @staticmethod
739
    def from_model_cls(model_cls: type[nn.Module]):
740
741
742
743
744
745
746
747
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

748
    def load_model_cls(self) -> type[nn.Module]:
749
750
751
752
753
754
755
756
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
757

758
759
760
    module_name: str
    class_name: str

761
762
763
764
765
766
767
768
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

769
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
770
771
        try:
            try:
772
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
773
774
775
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
776
                logger.debug(
777
                    "Cached model info file for class %s.%s not found",
778
779
780
                    self.module_name,
                    self.class_name,
                )
781
782
783
                return None

            if mi_dict["hash"] != module_hash:
784
                logger.debug(
785
                    "Cached model info file for class %s.%s is stale",
786
787
788
                    self.module_name,
                    self.class_name,
                )
789
790
791
792
793
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
794
            logger.debug(
795
                "Cached model info for class %s.%s error. ",
796
797
798
                self.module_name,
                self.class_name,
            )
799
800
            return None

801
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
802
803
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
804

805
806
807
808
809
810
811
812
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
813
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
814
815
816
817
818
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
819
    def inspect_model_cls(self) -> _ModelInfo:
820
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
821
        module_hash = None
822

823
824
        if model_path.exists():
            with open(model_path, "rb") as f:
825
                module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest()
826
827
828

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
829
                logger.debug(
830
                    "Loaded model info for class %s.%s from cache",
831
832
833
                    self.module_name,
                    self.class_name,
                )
834
835
                return mi
            else:
836
                logger.debug(
837
                    "Cache model info for class %s.%s miss. Loading model instead.",
838
839
840
                    self.module_name,
                    self.class_name,
                )
841
842
843

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
844
845
846
847
848
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
849
850

        # save cache file
851
852
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
853
854

        return mi
855

856
    def load_model_cls(self) -> type[nn.Module]:
857
858
859
860
861
862
863
864
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
865
) -> type[nn.Module] | None:
866
    from vllm.platforms import current_platform
867

868
    current_platform.verify_model_arch(model_arch)
869
870
871
    try:
        return model.load_model_cls()
    except Exception:
872
        logger.exception("Error in loading model architecture '%s'", model_arch)
873
        return None
874
875


876
877
878
879
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
880
) -> _ModelInfo | None:
881
882
883
    try:
        return model.inspect_model_cls()
    except Exception:
884
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
885
        return None
886
887


888
889
890
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
891
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
892

893
    def get_supported_archs(self) -> Set[str]:
894
        return self.models.keys()
895

896
897
898
    def register_model(
        self,
        model_arch: str,
899
        model_cls: type[nn.Module] | str,
900
    ) -> None:
901
902
903
        """
        Register an external model to be used in vLLM.

904
        `model_cls` can be either:
905

906
        - A [`torch.nn.Module`][] class directly referencing the model.
907
        - A string in the format `<module>:<class>` which can be used to
908
909
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
910
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
911
        """
912
913
914
915
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

916
        if model_arch in self.models:
917
918
            logger.warning(
                "Model architecture %s is already registered, and will be "
919
920
921
922
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
923
924
925
926
927
928

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
929

930
            model = _LazyRegisteredModel(*split_str)
931
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
932
            model = _RegisteredModel.from_model_cls(model_cls)
933
        else:
934
935
936
937
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
938
            raise TypeError(msg)
939

940
        self.models[model_arch] = model
941

942
    def _raise_for_unsupported(self, architectures: list[str]):
943
        all_supported_archs = self.get_supported_archs()
944

945
946
947
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
948
949
                "to be inspected. Please check the logs for more details."
            )
950

951
952
953
954
955
956
957
958
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
959
960
                    "use this model architecture."
                )
961

962
963
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
964
965
            f"Supported architectures: {all_supported_archs}"
        )
966

967
    def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None:
968
969
        if model_arch not in self.models:
            return None
970

971
        return _try_load_model_cls(model_arch, self.models[model_arch])
972

973
    def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None:
974
975
        if model_arch not in self.models:
            return None
976

977
978
979
980
981
982
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
983
    ) -> str | None:
984
985
986
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

987
988
989
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1006
                        trust_remote_code=model_config.trust_remote_code,
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
1019
                        trust_remote_code=model_config.trust_remote_code,
1020
1021
1022
1023
1024
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
1025
                if model_config.model_impl != "transformers":
1026
1027
1028
1029
1030
1031
1032
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
1033
1034
                    "'auto_map' (relevant if the model is custom)."
                )
1035
1036

        if not model_module.is_backend_compatible():
1037
            if model_config.model_impl != "transformers":
1038
                return None
1039

1040
1041
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
1042
1043
                "is not compatible with vLLM."
            )
1044

1045
        return model_config._get_transformers_backend_cls()
1046

1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
1072

1073
1074
    def inspect_model_cls(
        self,
1075
        architectures: str | list[str],
1076
        model_config: ModelConfig,
1077
    ) -> tuple[_ModelInfo, str]:
1078
1079
        if isinstance(architectures, str):
            architectures = [architectures]
1080
1081
        if not architectures:
            raise ValueError("No model architectures are specified")
1082
1083

        # Require transformers impl
1084
        if model_config.model_impl == "transformers":
1085
            arch = self._try_resolve_transformers(architectures[0], model_config)
1086
1087
1088
1089
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
1090
        elif model_config.model_impl == "terratorch":
1091
1092
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
1093

1094
        # Fallback to transformers impl (after resolving convert_type)
1095
1096
1097
1098
1099
1100
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1101
1102
1103
1104
1105
1106
1107
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1108
            model_info = self._try_inspect_model_cls(normalized_arch)
1109
            if model_info is not None:
1110
                return (model_info, arch)
1111

1112
        # Fallback to transformers impl (before resolving runner_type)
1113
1114
1115
1116
1117
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1118
1119
1120
1121
1122
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

1123
        return self._raise_for_unsupported(architectures)
1124

1125
1126
    def resolve_model_cls(
        self,
1127
        architectures: str | list[str],
1128
        model_config: ModelConfig,
1129
    ) -> tuple[type[nn.Module], str]:
1130
1131
        if isinstance(architectures, str):
            architectures = [architectures]
1132
1133
        if not architectures:
            raise ValueError("No model architectures are specified")
1134
1135

        # Require transformers impl
1136
        if model_config.model_impl == "transformers":
1137
            arch = self._try_resolve_transformers(architectures[0], model_config)
1138
1139
1140
1141
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
1142
        elif model_config.model_impl == "terratorch":
1143
1144
1145
1146
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
1147

1148
        # Fallback to transformers impl (after resolving convert_type)
1149
1150
1151
1152
1153
1154
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1155
1156
1157
1158
1159
1160
1161
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
1162
            model_cls = self._try_load_model_cls(normalized_arch)
1163
1164
            if model_cls is not None:
                return (model_cls, arch)
1165

1166
        # Fallback to transformers impl (before resolving runner_type)
1167
1168
1169
1170
1171
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
1172
1173
1174
1175
1176
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

1177
        return self._raise_for_unsupported(architectures)
1178

1179
1180
    def is_text_generation_model(
        self,
1181
        architectures: str | list[str],
1182
        model_config: ModelConfig,
1183
    ) -> bool:
1184
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1185
        return model_cls.is_text_generation_model
1186

1187
    def is_pooling_model(
1188
        self,
1189
        architectures: str | list[str],
1190
        model_config: ModelConfig,
1191
    ) -> bool:
1192
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1193
        return model_cls.is_pooling_model
1194
1195
1196

    def is_multimodal_model(
        self,
1197
        architectures: str | list[str],
1198
        model_config: ModelConfig,
1199
    ) -> bool:
1200
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1201
        return model_cls.supports_multimodal
1202

1203
    def is_multimodal_raw_input_only_model(
1204
        self,
1205
        architectures: str | list[str],
1206
        model_config: ModelConfig,
1207
    ) -> bool:
1208
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1209
        return model_cls.supports_multimodal_raw_input_only
1210

1211
1212
    def is_pp_supported_model(
        self,
1213
        architectures: str | list[str],
1214
        model_config: ModelConfig,
1215
    ) -> bool:
1216
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1217
        return model_cls.supports_pp
1218

1219
1220
    def model_has_inner_state(
        self,
1221
        architectures: str | list[str],
1222
        model_config: ModelConfig,
1223
    ) -> bool:
1224
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1225
        return model_cls.has_inner_state
1226

1227
1228
    def is_attention_free_model(
        self,
1229
        architectures: str | list[str],
1230
        model_config: ModelConfig,
1231
    ) -> bool:
1232
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1233
        return model_cls.is_attention_free
1234

1235
1236
    def is_hybrid_model(
        self,
1237
        architectures: str | list[str],
1238
        model_config: ModelConfig,
1239
    ) -> bool:
1240
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1241
1242
        return model_cls.is_hybrid

1243
1244
    def is_noops_model(
        self,
1245
        architectures: str | list[str],
1246
        model_config: ModelConfig,
1247
    ) -> bool:
1248
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1249
1250
        return model_cls.has_noops

1251
1252
    def is_transcription_model(
        self,
1253
        architectures: str | list[str],
1254
        model_config: ModelConfig,
1255
    ) -> bool:
1256
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1257
1258
        return model_cls.supports_transcription

1259
1260
    def is_transcription_only_model(
        self,
1261
        architectures: str | list[str],
1262
        model_config: ModelConfig,
1263
    ) -> bool:
1264
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1265
1266
        return model_cls.supports_transcription_only

1267

1268
1269
1270
1271
1272
1273
1274
1275
1276
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1277
1278
1279
1280
1281

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1282
1283
1284
1285
1286
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1287
        # `cloudpickle` allows pickling lambda functions directly
1288
        import cloudpickle
1289

1290
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1291
1292
1293

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1294
1295
1296
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1297
1298
1299
1300
1301
1302

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1303
1304
1305
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1306

1307
        with open(output_filepath, "rb") as f:
1308
1309
1310
1311
1312
1313
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1314

1315
1316
1317
1318
1319
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1320
1321
1322

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1323
1324
1325


if __name__ == "__main__":
1326
    _run()