registry.py 43.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
"""
Whenever you add an architecture to this page, please also update
`tests/models/registry.py` with example HuggingFace models for it.
"""
7

8
import hashlib
9
import importlib
10
import json
11
import os
12
import pickle
13
14
import subprocess
import sys
15
import tempfile
16
from abc import ABC, abstractmethod
17
from collections.abc import Set
18
from dataclasses import asdict, dataclass, field
19
from functools import lru_cache
20
from pathlib import Path
21
from typing import Callable, Optional, TypeVar, Union
22
23

import torch.nn as nn
24
import transformers
25

26
from vllm import envs
27
28
29
30
31
from vllm.config import (
    ModelConfig,
    iter_architecture_defaults,
    try_match_architecture_defaults,
)
32
from vllm.logger import init_logger
33
from vllm.logging_utils import logtime
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module

from .interfaces import (
    has_inner_state,
    has_noops,
    is_attention_free,
    is_hybrid,
    supports_cross_encoding,
    supports_multimodal,
    supports_multimodal_encoder_tp_data,
    supports_multimodal_raw_input_only,
    supports_pp,
    supports_transcription,
    supports_v0_only,
)
from .interfaces_base import (
    get_default_pooling_type,
    is_pooling_model,
    is_text_generation_model,
)
54
55
56

logger = init_logger(__name__)

57
58
_TEXT_GENERATION_MODELS = {
    # [Decoder-only]
59
    "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
60
61
    "AquilaModel": ("llama", "LlamaForCausalLM"),
    "AquilaForCausalLM": ("llama", "LlamaForCausalLM"),  # AquilaChat2
Raghav Ravishankar's avatar
Raghav Ravishankar committed
62
    "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
63
    "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
64
    "MiniMaxForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
65
    "MiniMaxText01ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
66
    "MiniMaxM1ForCausalLM": ("minimax_text_01", "MiniMaxText01ForCausalLM"),
67
68
69
70
    # baichuan-7b, upper case 'C' in the class name
    "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"),
    # baichuan-13b, lower case 'c' in the class name
    "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
71
    "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
ant-yy's avatar
ant-yy committed
72
    "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
Yu Chin Fabian Lim's avatar
Yu Chin Fabian Lim committed
73
    "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
74
    "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
75
    "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
76
    "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
77
    "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
78
    "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
79
    "CwmForCausalLM": ("llama", "LlamaForCausalLM"),
80
    "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
81
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
82
83
    "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
    "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
84
    "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
85
    "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
86
    "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
87
    "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
88
    "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
89
    "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
90
    "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
91
    "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
92
    "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
93
    "FlexOlmoForCausalLM": ("flex_olmo", "FlexOlmoForCausalLM"),
94
95
    "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
    "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
96
    "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
97
    "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
98
    "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
99
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
100
    "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
101
    "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
102
    "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
103
104
105
106
107
108
    "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
    "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
    "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
    "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
    "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
    "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
109
110
    "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),  # noqa: E501
    "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),  # noqa: E501
111
    "GritLM": ("gritlm", "GritLM"),
Michael Goin's avatar
Michael Goin committed
112
    "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
113
114
    "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
    "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
115
    "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
116
117
    "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
    "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
118
    "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
119
    "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
120
121
    "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
    "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
122
    "Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
Paul Pak's avatar
Paul Pak committed
123
    "Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
124
    "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
125
    "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
126
127
    # For decapoda-research/llama-*
    "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
XuruiYang's avatar
XuruiYang committed
128
    "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
129
    "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
130
    "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
131
    "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
132
    "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
133
134
    "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
    "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
135
136
137
138
139
    "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
    "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
    # transformers's mpt class has lower case
    "MptForCausalLM": ("mpt", "MPTForCausalLM"),
    "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
140
    "MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
141
    "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
Luis Vega's avatar
Luis Vega committed
142
    "NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
143
    "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
144
    "Olmo2ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
145
    "Olmo3ForCausalLM": ("olmo2", "Olmo2ForCausalLM"),
146
147
148
149
150
151
152
    "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
    "OPTForCausalLM": ("opt", "OPTForCausalLM"),
    "OrionForCausalLM": ("orion", "OrionForCausalLM"),
    "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
    "PhiForCausalLM": ("phi", "PhiForCausalLM"),
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
    "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
Shinichi Hemmi's avatar
Shinichi Hemmi committed
153
    "Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
154
    "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
155
156
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
    "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
157
158
    "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
    "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
159
    "RWForCausalLM": ("falcon", "FalconForCausalLM"),
160
    "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
Song's avatar
Song committed
161
    "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
162
163
164
165
    "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
    "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
    "SolarForCausalLM": ("solar", "SolarForCausalLM"),
166
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
167
    "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
168
    "XverseForCausalLM": ("llama", "LlamaForCausalLM"),
169
    "Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
170
171
172
}

_EMBEDDING_MODELS = {
173
    # [Text-only]
174
    "BertModel": ("bert", "BertEmbeddingModel"),
175
    "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
176
    "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
177
    "Gemma3TextModel": ("gemma3", "Gemma3Model"),
178
    "GlmForCausalLM": ("glm", "GlmForCausalLM"),
179
    "GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
180
    "GritLM": ("gritlm", "GritLM"),
181
182
    "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
    "GteNewModel": ("bert_with_rope", "GteNewModel"),
183
    "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
184
    "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),  # noqa: E501
185
    "LlamaModel": ("llama", "LlamaForCausalLM"),
186
187
    **{
        # Multiple models share the same architecture, so we include them all
188
189
        k: (mod, arch)
        for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
190
191
        if arch == "LlamaForCausalLM"
    },
192
    "MistralModel": ("llama", "LlamaForCausalLM"),
193
    "ModernBertModel": ("modernbert", "ModernBertModel"),
194
    "NomicBertModel": ("bert_with_rope", "NomicBertModel"),
195
    "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
196
    "Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
197
    "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
198
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
199
    "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
200
201
    "RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
    "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
202
    "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
203
    "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
204
    # [Multimodal]
205
    "CLIPModel": ("clip", "CLIPEmbeddingModel"),
206
207
208
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
209
    ),
210
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
211
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
212
213
    # Technically Terratorch models work on images, both in
    # input and output. I am adding it here because it piggy-backs on embedding
214
    # models for the time being.
215
216
    "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
    "Terratorch": ("terratorch", "Terratorch"),
217
218
}

219
220
_CROSS_ENCODER_MODELS = {
    "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
221
    "BertForTokenClassification": ("bert", "BertForTokenClassification"),
222
223
224
225
226
227
228
229
    "GteNewForSequenceClassification": (
        "bert_with_rope",
        "GteNewForSequenceClassification",
    ),
    "ModernBertForSequenceClassification": (
        "modernbert",
        "ModernBertForSequenceClassification",
    ),
230
231
232
233
    "ModernBertForTokenClassification": (
        "modernbert",
        "ModernBertForTokenClassification",
    ),
234
235
236
237
238
    "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
    "XLMRobertaForSequenceClassification": (
        "roberta",
        "RobertaForSequenceClassification",
    ),
239
    # [Auto-converted (see adapters.py)]
240
    "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),  # noqa: E501,
241
242
}

243
_MULTIMODAL_MODELS = {
244
    # [Decoder-only]
245
    "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
246
247
248
    "AyaVisionForConditionalGeneration": (
        "aya_vision",
        "AyaVisionForConditionalGeneration",
249
    ),
250
    "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
251
252
253
    "ChameleonForConditionalGeneration": (
        "chameleon",
        "ChameleonForConditionalGeneration",
254
    ),
255
256
257
    "Cohere2VisionForConditionalGeneration": (
        "cohere2_vision",
        "Cohere2VisionForConditionalGeneration",
258
    ),
259
    "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
Roger Wang's avatar
Roger Wang committed
260
    "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
261
262
263
    "Ernie4_5_VLMoeForConditionalGeneration": (
        "ernie45_vl",
        "Ernie4_5_VLMoeForConditionalGeneration",
264
    ),
265
    "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
266
    "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),  # noqa: E501
267
268
269
    "Gemma3nForConditionalGeneration": (
        "gemma3n_mm",
        "Gemma3nForConditionalGeneration",
270
    ),
271
    "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
272
    "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),  # noqa: E501
Jee Jee Li's avatar
Jee Jee Li committed
273
    "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),  # noqa: E501
274
275
276
    "GraniteSpeechForConditionalGeneration": (
        "granite_speech",
        "GraniteSpeechForConditionalGeneration",
277
    ),
278
    "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
279
    "InternVLChatModel": ("internvl", "InternVLChatModel"),
280
    "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
281
282
283
    "InternS1ForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
284
    ),
285
286
287
    "InternVLForConditionalGeneration": (
        "interns1",
        "InternS1ForConditionalGeneration",
288
    ),
289
290
291
292
293
    "Idefics3ForConditionalGeneration": (
        "idefics3",
        "Idefics3ForConditionalGeneration",
    ),
    "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),  # noqa: E501
294
    "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
295
296
297
    "KeyeVL1_5ForConditionalGeneration": (
        "keye_vl1_5",
        "KeyeVL1_5ForConditionalGeneration",
298
    ),
299
    "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
300
    "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),  # noqa: E501
301
    "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
302
    "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),  # noqa: E501
303
    "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
304
305
306
    "LlavaNextForConditionalGeneration": (
        "llava_next",
        "LlavaNextForConditionalGeneration",
307
    ),
308
309
310
    "LlavaNextVideoForConditionalGeneration": (
        "llava_next_video",
        "LlavaNextVideoForConditionalGeneration",
311
    ),
312
313
314
    "LlavaOnevisionForConditionalGeneration": (
        "llava_onevision",
        "LlavaOnevisionForConditionalGeneration",
315
    ),
316
    "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),  # noqa: E501
317
    "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
318
319
320
    "MiniMaxVL01ForConditionalGeneration": (
        "minimax_vl_01",
        "MiniMaxVL01ForConditionalGeneration",
321
    ),
322
    "MiniCPMO": ("minicpmo", "MiniCPMO"),
323
    "MiniCPMV": ("minicpmv", "MiniCPMV"),
324
325
326
    "Mistral3ForConditionalGeneration": (
        "mistral3",
        "Mistral3ForConditionalGeneration",
327
    ),
328
    "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
329
    "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
330
    "Ovis": ("ovis", "Ovis"),
331
    "Ovis2_5": ("ovis2_5", "Ovis2_5"),
332
333
334
    "PaliGemmaForConditionalGeneration": (
        "paligemma",
        "PaliGemmaForConditionalGeneration",
335
    ),
336
    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
337
338
    "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
    "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"),  # noqa: E501
339
    "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),  # noqa: E501
340
    "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),  # noqa: E501
341
    "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
342
343
344
    "Qwen2_5_VLForConditionalGeneration": (
        "qwen2_5_vl",
        "Qwen2_5_VLForConditionalGeneration",
345
    ),
346
347
348
    "Qwen2AudioForConditionalGeneration": (
        "qwen2_audio",
        "Qwen2AudioForConditionalGeneration",
349
    ),
350
351
352
    "Qwen2_5OmniModel": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
353
    ),
354
355
356
    "Qwen2_5OmniForConditionalGeneration": (
        "qwen2_5_omni_thinker",
        "Qwen2_5OmniThinkerForConditionalGeneration",
357
    ),
358
    "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),  # noqa: E501
359
360
361
    "Qwen3VLMoeForConditionalGeneration": (
        "qwen3_vl_moe",
        "Qwen3VLMoeForConditionalGeneration",
362
    ),
363
    "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
Song's avatar
Song committed
364
    "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),  # noqa: E501
汪志鹏's avatar
汪志鹏 committed
365
    "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),  # noqa: E501
366
367
368
    "Tarsier2ForConditionalGeneration": (
        "qwen2_vl",
        "Tarsier2ForConditionalGeneration",
369
    ),
370
    "UltravoxModel": ("ultravox", "UltravoxModel"),
Patrick von Platen's avatar
Patrick von Platen committed
371
    "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),  # noqa: E501
372
    # [Encoder-decoder]
373
    "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),  # noqa: E501
374
}
375
376

_SPECULATIVE_DECODING_MODELS = {
377
    "MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
378
    "EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
zhiweiz's avatar
zhiweiz committed
379
    "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
380
    "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
381
    "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
382
    "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
383
    "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
384
    "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
385
    "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
386
    "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
XuruiYang's avatar
XuruiYang committed
387
    "LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
Yuxuan Zhang's avatar
Yuxuan Zhang committed
388
    "Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
389
    "MedusaModel": ("medusa", "Medusa"),
390
    "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
391
392
393
    # Temporarily disabled.
    # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
    # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
394
}
395

396
_TRANSFORMERS_SUPPORTED_MODELS = {
397
398
399
    # Text generation models
    "SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
    # Multimodal models
400
401
402
403
    "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
}

_TRANSFORMERS_BACKEND_MODELS = {
404
    "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
405
    "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"),  # noqa: E501
406
    "TransformersMoEForCausalLM": ("transformers_moe", "TransformersMoEForCausalLM"),  # noqa: E501
407
408
409
    "TransformersMoEForMultimodalLM": (
        "transformers_moe",
        "TransformersMoEForMultimodalLM",
410
    ),
411
412
413
    "TransformersEmbeddingModel": (
        "transformers_pooling",
        "TransformersEmbeddingModel",
414
    ),
415
416
417
    "TransformersForSequenceClassification": (
        "transformers_pooling",
        "TransformersForSequenceClassification",
418
    ),
419
420
421
    "TransformersMoEForSequenceClassification": (
        "transformers_pooling",
        "TransformersMoEForSequenceClassification",
422
    ),
423
424
425
    "TransformersMoEEmbeddingModel": (
        "transformers_pooling",
        "TransformersMoEEmbeddingModel",
426
    ),
427
}
428

429
_VLLM_MODELS = {
430
    **_TEXT_GENERATION_MODELS,
431
    **_EMBEDDING_MODELS,
432
    **_CROSS_ENCODER_MODELS,
433
    **_MULTIMODAL_MODELS,
434
    **_SPECULATIVE_DECODING_MODELS,
435
436
    **_TRANSFORMERS_SUPPORTED_MODELS,
    **_TRANSFORMERS_BACKEND_MODELS,
437
438
}

439
440
441
442
# This variable is used as the args for subprocess.run(). We
# can modify  this variable to alter the args if needed. e.g.
# when we use par format to pack things together, sys.executable
# might not be the target we want to run.
443
_SUBPROCESS_COMMAND = [sys.executable, "-m", "vllm.model_executor.models.registry"]
444

445
_PREVIOUSLY_SUPPORTED_MODELS = {
446
    "MotifForCausalLM": "0.10.2",
447
    "Phi3SmallForCausalLM": "0.9.2",
448
    "Phi4FlashForCausalLM": "0.10.2",
449
450
451
452
453
454
455
456
457
    # encoder-decoder models except whisper
    # have been removed for V0 deprecation.
    "BartModel": "0.10.2",
    "BartForConditionalGeneration": "0.10.2",
    "DonutForConditionalGeneration": "0.10.2",
    "Florence2ForConditionalGeneration": "0.10.2",
    "MBartForConditionalGeneration": "0.10.2",
    "MllamaForConditionalGeneration": "0.10.2",
}
458

459

460
461
@dataclass(frozen=True)
class _ModelInfo:
462
    architecture: str
463
    is_text_generation_model: bool
464
    is_pooling_model: bool
465
    default_pooling_type: str
466
    supports_cross_encoding: bool
467
    supports_multimodal: bool
468
    supports_multimodal_raw_input_only: bool
469
    supports_multimodal_encoder_tp_data: bool
470
    supports_pp: bool
471
472
    has_inner_state: bool
    is_attention_free: bool
473
    is_hybrid: bool
474
    has_noops: bool
475
    supports_transcription: bool
476
    supports_transcription_only: bool
477
    supports_v0_only: bool
478
479

    @staticmethod
480
    def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
481
        return _ModelInfo(
482
            architecture=model.__name__,
483
            is_text_generation_model=is_text_generation_model(model),
484
            is_pooling_model=is_pooling_model(model),
485
            default_pooling_type=get_default_pooling_type(model),
486
            supports_cross_encoding=supports_cross_encoding(model),
487
            supports_multimodal=supports_multimodal(model),
488
489
490
491
492
493
            supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(
                model
            ),
            supports_multimodal_encoder_tp_data=supports_multimodal_encoder_tp_data(
                model
            ),
494
            supports_pp=supports_pp(model),
495
496
            has_inner_state=has_inner_state(model),
            is_attention_free=is_attention_free(model),
497
            is_hybrid=is_hybrid(model),
498
            supports_transcription=supports_transcription(model),
499
500
501
            supports_transcription_only=(
                supports_transcription(model) and model.supports_transcription_only
            ),
502
            supports_v0_only=supports_v0_only(model),
503
            has_noops=has_noops(model),
504
        )
505
506


507
508
509
510
class _BaseRegisteredModel(ABC):
    @abstractmethod
    def inspect_model_cls(self) -> _ModelInfo:
        raise NotImplementedError
511

512
    @abstractmethod
513
    def load_model_cls(self) -> type[nn.Module]:
514
        raise NotImplementedError
515
516


517
518
519
520
521
522
523
@dataclass(frozen=True)
class _RegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has already been imported in the main process.
    """

    interfaces: _ModelInfo
524
    model_cls: type[nn.Module]
525
526

    @staticmethod
527
    def from_model_cls(model_cls: type[nn.Module]):
528
529
530
531
532
533
534
535
        return _RegisteredModel(
            interfaces=_ModelInfo.from_model_cls(model_cls),
            model_cls=model_cls,
        )

    def inspect_model_cls(self) -> _ModelInfo:
        return self.interfaces

536
    def load_model_cls(self) -> type[nn.Module]:
537
538
539
540
541
542
543
544
        return self.model_cls


@dataclass(frozen=True)
class _LazyRegisteredModel(_BaseRegisteredModel):
    """
    Represents a model that has not been imported in the main process.
    """
545

546
547
548
    module_name: str
    class_name: str

549
550
551
552
553
554
555
556
    @staticmethod
    def _get_cache_dir() -> Path:
        return Path(envs.VLLM_CACHE_ROOT) / "modelinfos"

    def _get_cache_filename(self) -> str:
        cls_name = f"{self.module_name}-{self.class_name}".replace(".", "-")
        return f"{cls_name}.json"

557
    def _load_modelinfo_from_cache(self, module_hash: str) -> _ModelInfo | None:
558
559
        try:
            try:
560
                modelinfo_path = self._get_cache_dir() / self._get_cache_filename()
561
562
563
                with open(modelinfo_path, encoding="utf-8") as file:
                    mi_dict = json.load(file)
            except FileNotFoundError:
564
565
566
567
568
                logger.debug(
                    ("Cached model info file for class %s.%s not found"),
                    self.module_name,
                    self.class_name,
                )
569
570
571
                return None

            if mi_dict["hash"] != module_hash:
572
573
574
575
576
                logger.debug(
                    ("Cached model info file for class %s.%s is stale"),
                    self.module_name,
                    self.class_name,
                )
577
578
579
580
581
                return None

            # file not changed, use cached _ModelInfo properties
            return _ModelInfo(**mi_dict["modelinfo"])
        except Exception:
582
583
584
585
586
            logger.exception(
                ("Cached model info for class %s.%s error. "),
                self.module_name,
                self.class_name,
            )
587
588
            return None

589
    def _save_modelinfo_to_cache(self, mi: _ModelInfo, module_hash: str) -> None:
590
591
        """save dictionary json file to cache"""
        from vllm.model_executor.model_loader.weight_utils import atomic_writer
592

593
594
595
596
597
598
599
600
        try:
            modelinfo_dict = {
                "hash": module_hash,
                "modelinfo": asdict(mi),
            }
            cache_dir = self._get_cache_dir()
            cache_dir.mkdir(parents=True, exist_ok=True)
            modelinfo_path = cache_dir / self._get_cache_filename()
601
            with atomic_writer(modelinfo_path, encoding="utf-8") as f:
602
603
604
605
606
                json.dump(modelinfo_dict, f, indent=2)
        except Exception:
            logger.exception("Error saving model info cache.")

    @logtime(logger=logger, msg="Registry inspect model class")
607
    def inspect_model_cls(self) -> _ModelInfo:
608
        model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py"
609
        module_hash = None
610

611
612
        if model_path.exists():
            with open(model_path, "rb") as f:
613
                module_hash = hashlib.md5(f.read(), usedforsecurity=False).hexdigest()
614
615
616

            mi = self._load_modelinfo_from_cache(module_hash)
            if mi is not None:
617
618
619
620
621
                logger.debug(
                    ("Loaded model info for class %s.%s from cache"),
                    self.module_name,
                    self.class_name,
                )
622
623
                return mi
            else:
624
625
626
627
628
                logger.debug(
                    ("Cache model info for class %s.%s miss. Loading model instead."),
                    self.module_name,
                    self.class_name,
                )
629
630
631

        # Performed in another process to avoid initializing CUDA
        mi = _run_in_subprocess(
632
633
634
635
636
            lambda: _ModelInfo.from_model_cls(self.load_model_cls())
        )
        logger.debug(
            "Loaded model info for class %s.%s", self.module_name, self.class_name
        )
637
638

        # save cache file
639
640
        if module_hash is not None:
            self._save_modelinfo_to_cache(mi, module_hash)
641
642

        return mi
643

644
    def load_model_cls(self) -> type[nn.Module]:
645
646
647
648
649
650
651
652
        mod = importlib.import_module(self.module_name)
        return getattr(mod, self.class_name)


@lru_cache(maxsize=128)
def _try_load_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
653
) -> Optional[type[nn.Module]]:
654
    from vllm.platforms import current_platform
655

656
    current_platform.verify_model_arch(model_arch)
657
658
659
    try:
        return model.load_model_cls()
    except Exception:
660
        logger.exception("Error in loading model architecture '%s'", model_arch)
661
        return None
662
663


664
665
666
667
668
669
670
671
@lru_cache(maxsize=128)
def _try_inspect_model_cls(
    model_arch: str,
    model: _BaseRegisteredModel,
) -> Optional[_ModelInfo]:
    try:
        return model.inspect_model_cls()
    except Exception:
672
        logger.exception("Error in inspecting model architecture '%s'", model_arch)
673
        return None
674
675


676
677
678
@dataclass
class _ModelRegistry:
    # Keyed by model_arch
679
    models: dict[str, _BaseRegisteredModel] = field(default_factory=dict)
680

681
    def get_supported_archs(self) -> Set[str]:
682
        return self.models.keys()
683

684
685
686
    def register_model(
        self,
        model_arch: str,
687
        model_cls: Union[type[nn.Module], str],
688
    ) -> None:
689
690
691
        """
        Register an external model to be used in vLLM.

692
        `model_cls` can be either:
693

694
        - A [`torch.nn.Module`][] class directly referencing the model.
695
        - A string in the format `<module>:<class>` which can be used to
696
697
          lazily import the model. This is useful to avoid initializing CUDA
          when importing the model and thus the related error
698
          `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
699
        """
700
701
702
703
        if not isinstance(model_arch, str):
            msg = f"`model_arch` should be a string, not a {type(model_arch)}"
            raise TypeError(msg)

704
        if model_arch in self.models:
705
706
            logger.warning(
                "Model architecture %s is already registered, and will be "
707
708
709
710
                "overwritten by the new model class %s.",
                model_arch,
                model_cls,
            )
711
712
713
714
715
716

        if isinstance(model_cls, str):
            split_str = model_cls.split(":")
            if len(split_str) != 2:
                msg = "Expected a string in the format `<module>:<class>`"
                raise ValueError(msg)
717

718
            model = _LazyRegisteredModel(*split_str)
719
        elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
720
            model = _RegisteredModel.from_model_cls(model_cls)
721
        else:
722
723
724
725
            msg = (
                "`model_cls` should be a string or PyTorch model class, "
                f"not a {type(model_arch)}"
            )
726
            raise TypeError(msg)
727

728
        self.models[model_arch] = model
729

730
    def _raise_for_unsupported(self, architectures: list[str]):
731
        all_supported_archs = self.get_supported_archs()
732

733
734
735
        if any(arch in all_supported_archs for arch in architectures):
            raise ValueError(
                f"Model architectures {architectures} failed "
736
737
                "to be inspected. Please check the logs for more details."
            )
738

739
740
741
742
743
744
745
746
        for arch in architectures:
            if arch in _PREVIOUSLY_SUPPORTED_MODELS:
                previous_version = _PREVIOUSLY_SUPPORTED_MODELS[arch]

                raise ValueError(
                    f"Model architecture {arch} was supported in vLLM until "
                    f"v{previous_version}, and is not supported anymore. "
                    "Please use an older version of vLLM if you want to "
747
748
                    "use this model architecture."
                )
749

750
751
        raise ValueError(
            f"Model architectures {architectures} are not supported for now. "
752
753
            f"Supported architectures: {all_supported_archs}"
        )
754

755
    def _try_load_model_cls(self, model_arch: str) -> Optional[type[nn.Module]]:
756
757
        if model_arch not in self.models:
            return None
758

759
        return _try_load_model_cls(model_arch, self.models[model_arch])
760

761
    def _try_inspect_model_cls(self, model_arch: str) -> Optional[_ModelInfo]:
762
763
        if model_arch not in self.models:
            return None
764

765
766
767
768
769
770
771
772
773
774
        return _try_inspect_model_cls(model_arch, self.models[model_arch])

    def _try_resolve_transformers(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> Optional[str]:
        if architecture in _TRANSFORMERS_BACKEND_MODELS:
            return architecture

775
776
777
        auto_map: dict[str, str] = (
            getattr(model_config.hf_config, "auto_map", None) or dict()
        )
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810

        # Make sure that config class is always initialized before model class,
        # otherwise the model class won't be able to access the config class,
        # the expected auto_map should have correct order like:
        # "auto_map": {
        #     "AutoConfig": "<your-repo-name>--<config-name>",
        #     "AutoModel": "<your-repo-name>--<config-name>",
        #     "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
        # },
        for prefix in ("AutoConfig", "AutoModel"):
            for name, module in auto_map.items():
                if name.startswith(prefix):
                    try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=False,
                    )

        model_module = getattr(transformers, architecture, None)

        if model_module is None:
            for name, module in auto_map.items():
                if name.startswith("AutoModel"):
                    model_module = try_get_class_from_dynamic_module(
                        module,
                        model_config.model,
                        revision=model_config.revision,
                        warn_on_fail=True,
                    )
                    if model_module is not None:
                        break
            else:
811
                if model_config.model_impl != "transformers":
812
813
814
815
816
817
818
                    return None

                raise ValueError(
                    f"Cannot find model module. {architecture!r} is not a "
                    "registered model in the Transformers library (only "
                    "relevant if the model is meant to be in Transformers) "
                    "and 'AutoModel' is not present in the model config's "
819
820
                    "'auto_map' (relevant if the model is custom)."
                )
821
822

        if not model_module.is_backend_compatible():
823
            if model_config.model_impl != "transformers":
824
                return None
825

826
827
            raise ValueError(
                f"The Transformers implementation of {architecture!r} "
828
829
                "is not compatible with vLLM."
            )
830

831
        return model_config._get_transformers_backend_cls()
832

833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
    def _normalize_arch(
        self,
        architecture: str,
        model_config: ModelConfig,
    ) -> str:
        if architecture in self.models:
            return architecture

        # This may be called in order to resolve runner_type and convert_type
        # in the first place, in which case we consider the default match
        match = try_match_architecture_defaults(
            architecture,
            runner_type=getattr(model_config, "runner_type", None),
            convert_type=getattr(model_config, "convert_type", None),
        )
        if match:
            suffix, _ = match

            # Get the name of the base model to convert
            for repl_suffix, _ in iter_architecture_defaults():
                base_arch = architecture.replace(suffix, repl_suffix)
                if base_arch in self.models:
                    return base_arch

        return architecture
858

859
860
    def inspect_model_cls(
        self,
861
        architectures: Union[str, list[str]],
862
        model_config: ModelConfig,
863
    ) -> tuple[_ModelInfo, str]:
864
865
        if isinstance(architectures, str):
            architectures = [architectures]
866
867
        if not architectures:
            raise ValueError("No model architectures are specified")
868
869

        # Require transformers impl
870
        if model_config.model_impl == "transformers":
871
            arch = self._try_resolve_transformers(architectures[0], model_config)
872
873
874
875
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)
876
        elif model_config.model_impl == "terratorch":
877
878
            model_info = self._try_inspect_model_cls("Terratorch")
            return (model_info, "Terratorch")
879

880
        # Fallback to transformers impl (after resolving convert_type)
881
882
883
884
885
886
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
887
888
889
890
891
892
893
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
894
            model_info = self._try_inspect_model_cls(normalized_arch)
895
            if model_info is not None:
896
                return (model_info, arch)
897

898
        # Fallback to transformers impl (before resolving runner_type)
899
900
901
902
903
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
904
905
906
907
908
            if arch is not None:
                model_info = self._try_inspect_model_cls(arch)
                if model_info is not None:
                    return (model_info, arch)

909
        return self._raise_for_unsupported(architectures)
910

911
912
    def resolve_model_cls(
        self,
913
        architectures: Union[str, list[str]],
914
        model_config: ModelConfig,
915
    ) -> tuple[type[nn.Module], str]:
916
917
        if isinstance(architectures, str):
            architectures = [architectures]
918
919
        if not architectures:
            raise ValueError("No model architectures are specified")
920
921

        # Require transformers impl
922
        if model_config.model_impl == "transformers":
923
            arch = self._try_resolve_transformers(architectures[0], model_config)
924
925
926
927
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)
928
        elif model_config.model_impl == "terratorch":
929
930
931
932
            arch = "Terratorch"
            model_cls = self._try_load_model_cls(arch)
            if model_cls is not None:
                return (model_cls, arch)
933

934
        # Fallback to transformers impl (after resolving convert_type)
935
936
937
938
939
940
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
            and getattr(model_config, "convert_type", "none") == "none"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
941
942
943
944
945
946
947
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

        for arch in architectures:
            normalized_arch = self._normalize_arch(arch, model_config)
948
            model_cls = self._try_load_model_cls(normalized_arch)
949
950
            if model_cls is not None:
                return (model_cls, arch)
951

952
        # Fallback to transformers impl (before resolving runner_type)
953
954
955
956
957
        if (
            all(arch not in self.models for arch in architectures)
            and model_config.model_impl == "auto"
        ):
            arch = self._try_resolve_transformers(architectures[0], model_config)
958
959
960
961
962
            if arch is not None:
                model_cls = self._try_load_model_cls(arch)
                if model_cls is not None:
                    return (model_cls, arch)

963
        return self._raise_for_unsupported(architectures)
964

965
966
    def is_text_generation_model(
        self,
967
        architectures: Union[str, list[str]],
968
        model_config: ModelConfig,
969
    ) -> bool:
970
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
971
        return model_cls.is_text_generation_model
972

973
    def is_pooling_model(
974
        self,
975
        architectures: Union[str, list[str]],
976
        model_config: ModelConfig,
977
    ) -> bool:
978
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
979
        return model_cls.is_pooling_model
980

981
982
    def is_cross_encoder_model(
        self,
983
        architectures: Union[str, list[str]],
984
        model_config: ModelConfig,
985
    ) -> bool:
986
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
987
        return model_cls.supports_cross_encoding
988

989
990
    def is_multimodal_model(
        self,
991
        architectures: Union[str, list[str]],
992
        model_config: ModelConfig,
993
    ) -> bool:
994
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
995
        return model_cls.supports_multimodal
996

997
    def is_multimodal_raw_input_only_model(
998
999
        self,
        architectures: Union[str, list[str]],
1000
        model_config: ModelConfig,
1001
    ) -> bool:
1002
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1003
        return model_cls.supports_multimodal_raw_input_only
1004

1005
1006
    def is_pp_supported_model(
        self,
1007
        architectures: Union[str, list[str]],
1008
        model_config: ModelConfig,
1009
    ) -> bool:
1010
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1011
        return model_cls.supports_pp
1012

1013
1014
    def model_has_inner_state(
        self,
1015
        architectures: Union[str, list[str]],
1016
        model_config: ModelConfig,
1017
    ) -> bool:
1018
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1019
        return model_cls.has_inner_state
1020

1021
1022
    def is_attention_free_model(
        self,
1023
        architectures: Union[str, list[str]],
1024
        model_config: ModelConfig,
1025
    ) -> bool:
1026
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1027
        return model_cls.is_attention_free
1028

1029
1030
    def is_hybrid_model(
        self,
1031
        architectures: Union[str, list[str]],
1032
        model_config: ModelConfig,
1033
    ) -> bool:
1034
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1035
1036
        return model_cls.is_hybrid

1037
1038
    def is_noops_model(
        self,
1039
        architectures: Union[str, list[str]],
1040
        model_config: ModelConfig,
1041
    ) -> bool:
1042
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1043
1044
        return model_cls.has_noops

1045
1046
    def is_transcription_model(
        self,
1047
        architectures: Union[str, list[str]],
1048
        model_config: ModelConfig,
1049
    ) -> bool:
1050
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1051
1052
        return model_cls.supports_transcription

1053
1054
1055
    def is_transcription_only_model(
        self,
        architectures: Union[str, list[str]],
1056
        model_config: ModelConfig,
1057
    ) -> bool:
1058
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1059
1060
        return model_cls.supports_transcription_only

1061
1062
    def is_v1_compatible(
        self,
1063
        architectures: Union[str, list[str]],
1064
        model_config: ModelConfig,
1065
    ) -> bool:
1066
        model_cls, _ = self.inspect_model_cls(architectures, model_config)
1067
1068
        return not model_cls.supports_v0_only

1069

1070
1071
1072
1073
1074
1075
1076
1077
1078
ModelRegistry = _ModelRegistry(
    {
        model_arch: _LazyRegisteredModel(
            module_name=f"vllm.model_executor.models.{mod_relname}",
            class_name=cls_name,
        )
        for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items()
    }
)
1079
1080
1081
1082
1083

_T = TypeVar("_T")


def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
1084
1085
1086
1087
1088
    # NOTE: We use a temporary directory instead of a temporary file to avoid
    # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
    with tempfile.TemporaryDirectory() as tempdir:
        output_filepath = os.path.join(tempdir, "registry_output.tmp")

1089
        # `cloudpickle` allows pickling lambda functions directly
1090
        import cloudpickle
1091

1092
        input_bytes = cloudpickle.dumps((fn, output_filepath))
1093
1094
1095

        # cannot use `sys.executable __file__` here because the script
        # contains relative imports
1096
1097
1098
        returned = subprocess.run(
            _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True
        )
1099
1100
1101
1102
1103
1104

        # check if the subprocess is successful
        try:
            returned.check_returncode()
        except Exception as e:
            # wrap raised exception to provide more information
1105
1106
1107
            raise RuntimeError(
                f"Error raised in subprocess:\n{returned.stderr.decode()}"
            ) from e
1108

1109
        with open(output_filepath, "rb") as f:
1110
1111
1112
1113
1114
1115
            return pickle.load(f)


def _run() -> None:
    # Setup plugins
    from vllm.plugins import load_general_plugins
1116

1117
1118
1119
1120
1121
    load_general_plugins()

    fn, output_file = pickle.loads(sys.stdin.buffer.read())

    result = fn()
1122
1123
1124

    with open(output_file, "wb") as f:
        f.write(pickle.dumps(result))
1125
1126
1127


if __name__ == "__main__":
1128
    _run()