registry.py 38.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Mapping, Set
5
from dataclasses import dataclass, field
6
from typing import Any, Literal
7
8
9
10

import pytest
from packaging.version import Version
from transformers import __version__ as TRANSFORMERS_VERSION
11

12
from vllm.config.model import ModelDType, TokenizerMode
13

14
15
16
17
18
19
20
21
22

@dataclass(frozen=True)
class _HfExamplesInfo:
    default: str
    """The default model to use for testing this architecture."""

    extras: Mapping[str, str] = field(default_factory=dict)
    """Extra models to use for testing this architecture."""

23
    tokenizer: str | None = None
24
25
    """Set the tokenizer to load for this architecture."""

26
    tokenizer_mode: TokenizerMode | str = "auto"
27
28
    """Set the tokenizer type for this architecture."""

29
    speculative_model: str | None = None
30
31
32
33
34
    """
    The default model to use for testing this architecture, which is only used
    for speculative decoding.
    """

35
36
37
38
39
    speculative_method: str | None = None
    """
    The method to use for speculative decoding.
    """

40
    min_transformers_version: str | None = None
41
42
43
44
    """
    The minimum version of HF Transformers that is required to run this model.
    """

45
    max_transformers_version: str | None = None
46
47
48
49
    """
    The maximum version of HF Transformers that this model runs on.
    """

50
    transformers_version_reason: str | None = None
51
52
53
54
    """
    The reason for the minimum/maximum version requirement.
    """

55
    require_embed_inputs: bool = False
56
    """
57
58
    If `True`, enables prompt and multi-modal embedding inputs while
    disabling tokenization.
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    """

    dtype: ModelDType = "auto"
    """
    The data type for the model weights and activations.
    """

    enforce_eager: bool = False
    """
    Whether to enforce eager execution. If True, we will
    disable CUDA graph and always execute the model in eager mode.
    If False, we will use CUDA graph and eager execution in hybrid.
    """

73
74
    is_available_online: bool = True
    """
75
    Set this to `False` if the name of this architecture no longer exists on
76
77
78
79
80
81
    the HF repo. To maintain backwards compatibility, we have not removed them
    from the main model registry, so without this flag the registry tests will
    fail.
    """

    trust_remote_code: bool = False
82
    """The `trust_remote_code` level required to load the model."""
83

84
    hf_overrides: dict[str, Any] = field(default_factory=dict)
85
    """The `hf_overrides` required to load the model."""
86

87
    max_model_len: int | None = None
88
89
90
91
92
    """
    The maximum model length to use for this model. Some models default to a
    length that is too large to fit into memory in CI.
    """

93
    revision: str | None = None
94
95
96
97
98
    """
    The specific revision (commit hash, tag, or branch) to use for the model.
    If not specified, the default revision will be used.
    """

99
    max_num_seqs: int | None = None
100
101
    """Maximum number of sequences to be processed in a single iteration."""

102
103
104
105
106
107
    use_original_num_layers: bool = False
    """
    If True, use the original number of layers from the model config 
    instead of minimal layers for testing.
    """

108
109
110
    def check_transformers_version(
        self,
        *,
111
        on_fail: Literal["error", "skip", "return"],
112
113
        check_min_version: bool = True,
        check_max_version: bool = True,
114
    ) -> str | None:
115
116
117
118
        """
        If the installed transformers version does not meet the requirements,
        perform the given action.
        """
119
120
121
122
        if (
            self.min_transformers_version is None
            and self.max_transformers_version is None
        ):
123
            return None
124
125

        current_version = TRANSFORMERS_VERSION
126
        cur_base_version = Version(current_version).base_version
127
128
129
        min_version = self.min_transformers_version
        max_version = self.max_transformers_version
        msg = f"`transformers=={current_version}` installed, but `transformers"
130
131
        # Only check the base version for the min/max version, otherwise preview
        # models cannot be run because `x.yy.0.dev0`<`x.yy.0`
132
133
134
135
136
        if (
            check_min_version
            and min_version
            and Version(cur_base_version) < Version(min_version)
        ):
137
            msg += f">={min_version}` is required to run this model."
138
139
140
141
142
        elif (
            check_max_version
            and max_version
            and Version(cur_base_version) > Version(max_version)
        ):
143
144
            msg += f"<={max_version}` is required to run this model."
        else:
145
            return None
146

147
148
149
150
151
        if self.transformers_version_reason:
            msg += f" Reason: {self.transformers_version_reason}"

        if on_fail == "error":
            raise RuntimeError(msg)
152
        elif on_fail == "skip":
153
            pytest.skip(msg)
154

155
156
        return msg

157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    def check_available_online(
        self,
        *,
        on_fail: Literal["error", "skip"],
    ) -> None:
        """
        If the model is not available online, perform the given action.
        """
        if not self.is_available_online:
            msg = "Model is not available online"

            if on_fail == "error":
                raise RuntimeError(msg)
            else:
                pytest.skip(msg)

173
174
175

_TEXT_GENERATION_EXAMPLE_MODELS = {
    # [Decoder-only]
176
177
178
179
    "AfmoeForCausalLM": _HfExamplesInfo(
        "arcee-ai/Trinity-Nano",
        is_available_online=False,
    ),
180
    "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
181
182
    "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B", trust_remote_code=True),
    "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B", trust_remote_code=True),
183
    "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
    "ArcticForCausalLM": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-instruct", trust_remote_code=True
    ),
    "BaiChuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan-7B", trust_remote_code=True
    ),
    "BaichuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan2-7B-chat", trust_remote_code=True
    ),
    "BailingMoeForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-lite-1.5", trust_remote_code=True
    ),
    "BailingMoeV2ForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-mini-2.0", trust_remote_code=True
    ),
    "BambaForCausalLM": _HfExamplesInfo(
        "ibm-ai-platform/Bamba-9B-v1",
        extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"},
    ),
    "BloomForCausalLM": _HfExamplesInfo(
        "bigscience/bloom-560m", {"1b": "bigscience/bloomz-1b1"}
    ),
    "ChatGLMModel": _HfExamplesInfo(
        "zai-org/chatglm3-6b", trust_remote_code=True, max_transformers_version="4.48"
    ),
    "ChatGLMForConditionalGeneration": _HfExamplesInfo(
        "thu-coai/ShieldLM-6B-chatglm3",
        trust_remote_code=True,
    ),
    "CohereForCausalLM": _HfExamplesInfo(
        "CohereForAI/c4ai-command-r-v01", trust_remote_code=True
    ),
    "Cohere2ForCausalLM": _HfExamplesInfo(
        "CohereForAI/c4ai-command-r7b-12-2024",
        trust_remote_code=True,
    ),
220
    "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
221
    "DbrxForCausalLM": _HfExamplesInfo("databricks/dbrx-instruct"),
222
223
224
225
    "DeciLMForCausalLM": _HfExamplesInfo(
        "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
        trust_remote_code=True,
    ),
226
227
228
229
    "DeepseekForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-moe-16b-base",
        trust_remote_code=True,
    ),
230
231
232
233
234
235
236
237
    "DeepseekV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V2-Lite-Chat",
        trust_remote_code=True,
    ),
    "DeepseekV3ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V3",
        trust_remote_code=True,
    ),
238
    "DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
239
240
    "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"),
    "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"),
241
242
243
    "ExaoneForCausalLM": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
    ),
244
    "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
245
    "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
246
    "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
247
    "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
248
    "FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
249
    "GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
250
251
252
    "Gemma2ForCausalLM": _HfExamplesInfo(
        "google/gemma-2-9b", extras={"tiny": "google/gemma-2-2b-it"}
    ),
253
    "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
254
    "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
255
256
    "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
    "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
257
    "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
258
259
260
    "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
    "GPTBigCodeForCausalLM": _HfExamplesInfo(
        "bigcode/starcoder",
261
262
263
264
        extras={
            "tiny": "bigcode/tiny_starcoder_py",
            "santacoder": "bigcode/gpt_bigcode-santacoder",
        },
265
266
267
268
269
270
271
    ),
    "GPTJForCausalLM": _HfExamplesInfo(
        "Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"}
    ),
    "GPTNeoXForCausalLM": _HfExamplesInfo(
        "EleutherAI/pythia-70m", {"1b": "EleutherAI/pythia-1.4b"}
    ),
272
    "GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"),
273
274
    "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
    "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
275
    "GraniteMoeHybridForCausalLM": _HfExamplesInfo(
276
        "ibm-granite/granite-4.0-tiny-preview"
277
278
279
280
281
282
283
    ),
    "GraniteMoeSharedForCausalLM": _HfExamplesInfo(
        "ibm-research/moe-7b-1b-active-shared-experts"
    ),
    "Grok1ModelForCausalLM": _HfExamplesInfo(
        "hpcai-tech/grok-1", trust_remote_code=True
    ),
284
    "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
    "HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
        "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
    ),
    "InternLMForCausalLM": _HfExamplesInfo(
        "internlm/internlm-chat-7b", trust_remote_code=True
    ),
    "InternLM2ForCausalLM": _HfExamplesInfo(
        "internlm/internlm2-chat-7b", trust_remote_code=True
    ),
    "InternLM2VEForCausalLM": _HfExamplesInfo(
        "OpenGVLab/Mono-InternVL-2B", trust_remote_code=True
    ),
    "InternLM3ForCausalLM": _HfExamplesInfo(
        "internlm/internlm3-8b-instruct", trust_remote_code=True
    ),
300
    "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
301
302
303
304
305
306
307
    "JambaForCausalLM": _HfExamplesInfo(
        "ai21labs/AI21-Jamba-1.5-Mini",
        extras={
            "tiny": "ai21labs/Jamba-tiny-dev",
            "random": "ai21labs/Jamba-tiny-random",
        },
    ),
308
309
310
    "KimiLinearForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True
    ),
311
    "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
Paul Pak's avatar
Paul Pak committed
312
313
314
    "Lfm2MoeForCausalLM": _HfExamplesInfo(
        "LiquidAI/LFM2-8B-A1B", min_transformers_version="4.58"
    ),
315
316
317
318
319
320
    "LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.2-1B-Instruct",
        extras={
            "guard": "meta-llama/Llama-Guard-3-1B",
            "hermes": "NousResearch/Hermes-3-Llama-3.1-8B",
            "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
321
            "tiny": "hmellor/tiny-random-LlamaForCausalLM",
322
323
324
325
326
327
328
329
330
331
332
        },
    ),
    "LLaMAForCausalLM": _HfExamplesInfo(
        "decapoda-research/llama-7b-hf", is_available_online=False
    ),
    "Llama4ForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "LongcatFlashForCausalLM": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
    ),
333
    "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
334
335
336
337
338
339
340
341
342
343
344
345
346
    "Mamba2ForCausalLM": _HfExamplesInfo(
        "mistralai/Mamba-Codestral-7B-v0.1",
        extras={
            "random": "yujiepan/mamba2-codestral-v0.1-tiny-random",
        },
    ),
    "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"),
    "MiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-2B-sft-bf16", trust_remote_code=True
    ),
    "MiniCPM3ForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM3-4B", trust_remote_code=True
    ),
347
    "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
348
349
350
351
352
353
354
355
    "MiniMaxText01ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-Text-01",
        trust_remote_code=True,
        revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3",
    ),
    "MiniMaxM1ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-M1-40k", trust_remote_code=True
    ),
356
    "MiniMaxM2ForCausalLM": _HfExamplesInfo(
youkaichao's avatar
youkaichao committed
357
358
        "MiniMaxAI/MiniMax-M2",
        trust_remote_code=True,
359
    ),
360
    "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
361
362
363
    "MistralLarge3ForCausalLM": _HfExamplesInfo(
        "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", is_available_online=False
    ),
364
365
366
367
    "MixtralForCausalLM": _HfExamplesInfo(
        "mistralai/Mixtral-8x7B-Instruct-v0.1",
        {"tiny": "TitanML/tiny-mixtral"},
    ),
368
369
370
    "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
    "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b"),
    "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
371
372
373
    "NemotronHForCausalLM": _HfExamplesInfo(
        "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
    ),
374
    "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
375
    "Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
376
    "Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
377
    "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
378
379
380
381
382
    "OpenPanguMTPModel": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
    ),
383
384
385
386
387
388
    "OPTForCausalLM": _HfExamplesInfo(
        "facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
    ),
    "OrionForCausalLM": _HfExamplesInfo(
        "OrionStarAI/Orion-14B-Chat", trust_remote_code=True
    ),
389
    "OuroForCausalLM": _HfExamplesInfo("ByteDance/Ouro-1.4B", trust_remote_code=True),
390
391
392
393
394
395
396
397
    "PanguEmbeddedForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Embedded-7B-V1.1", trust_remote_code=True
    ),
    "PanguUltraMoEForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
    ),
398
    "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
399
    "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
400
    "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
401
402
403
404
405
406
407
    "PhiMoEForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3.5-MoE-instruct", trust_remote_code=True
    ),
    "Plamo2ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-2-1b",
        trust_remote_code=True,
    ),
408
409
410
411
    "Plamo3ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-3-nict-2b-base",
        trust_remote_code=True,
    ),
412
413
414
415
416
417
418
419
420
    "QWenLMHeadModel": _HfExamplesInfo(
        "Qwen/Qwen-7B-Chat",
        max_transformers_version="4.53",
        transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers",  # noqa: E501
        trust_remote_code=True,
    ),
    "Qwen2ForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen2-0.5B-Instruct", extras={"2.5": "Qwen/Qwen2.5-0.5B-Instruct"}
    ),
421
    "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
Jee Jee Li's avatar
Jee Jee Li committed
422
423
    "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
    "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
424
425
426
427
428
    "Qwen3NextForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct",
        extras={"tiny-random": "tiny-random/qwen3-next-moe"},
        min_transformers_version="4.56.3",
    ),
429
    "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
430
431
432
433
    "SeedOssForCausalLM": _HfExamplesInfo(
        "ByteDance-Seed/Seed-OSS-36B-Instruct",
        trust_remote_code=True,
    ),
434
    "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
435
    "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
436
    "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
437
    "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
438
439
440
441
    "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3", trust_remote_code=True),
    "SolarForCausalLM": _HfExamplesInfo(
        "upstage/solar-pro-preview-instruct", trust_remote_code=True
    ),
442
443
444
    "TeleChatForCausalLM": _HfExamplesInfo(
        "chuhac/TeleChat2-35B", trust_remote_code=True
    ),
445
446
447
448
449
450
451
452
453
454
455
    "TeleChat2ForCausalLM": _HfExamplesInfo(
        "Tele-AI/TeleChat2-3B", trust_remote_code=True
    ),
    "TeleFLMForCausalLM": _HfExamplesInfo(
        "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
    ),
    "XverseForCausalLM": _HfExamplesInfo(
        "xverse/XVERSE-7B-Chat",
        tokenizer="meta-llama/Llama-2-7b",
        trust_remote_code=True,
    ),
456
    "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
457
    "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
458
    "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
459
460
461
462
}

_EMBEDDING_EXAMPLE_MODELS = {
    # [Text-only]
463
    "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
464
    "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
465
    "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
466
    "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
467
468
469
470
471
472
473
474
475
476
477
478
    "GteModel": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True
    ),
    "GteNewModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-base-en-v1.5",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewModel"]},
    ),
    "InternLM2ForRewardModel": _HfExamplesInfo(
        "internlm/internlm2-1_8b-reward", trust_remote_code=True
    ),
    "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"),
479
    "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
480
    "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
481
482
483
484
485
486
    "ModernBertModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-modernbert-base", trust_remote_code=True
    ),
    "NomicBertModel": _HfExamplesInfo(
        "nomic-ai/nomic-embed-text-v2-moe", trust_remote_code=True
    ),
487
    "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
488
489
490
491
492
493
494
495
496
497
498
499
500
    "Qwen2ForRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-RM-72B",
        max_transformers_version="4.53",
        transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers",  # noqa: E501
    ),
    "Qwen2ForProcessRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-PRM-7B",
        max_transformers_version="4.53",
        transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers",  # noqa: E501
    ),
    "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"),
    "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"),
    "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"),
501
    "BertSpladeSparseEmbeddingModel": _HfExamplesInfo(
502
503
        "naver/splade-v3",
        hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
504
    ),
505
    # [Multimodal]
506
    "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
507
    "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
508
509
510
511
    "Phi3VForCausalLM": _HfExamplesInfo(
        "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
512
    "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"),
513
514
    "PrithviGeoSpatialMAE": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
515
        dtype="float16",
516
        enforce_eager=True,
517
518
        require_embed_inputs=True,
        # This is to avoid the model going OOM in CI
519
520
521
522
        max_num_seqs=32,
    ),
    "Terratorch": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
523
        dtype="float16",
524
        enforce_eager=True,
525
        require_embed_inputs=True,
526
527
528
        # This is to avoid the model going OOM in CI
        max_num_seqs=32,
    ),
529
530
}

531
532
_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
    # [Decoder-only]
533
534
535
    "GPT2ForSequenceClassification": _HfExamplesInfo(
        "nie3e/sentiment-polish-gpt2-small"
    ),
536
    # [Cross-encoder]
537
538
539
    "BertForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/ms-marco-MiniLM-L-6-v2"
    ),
540
    "BertForTokenClassification": _HfExamplesInfo("boltuix/NeuroBERT-NER"),
541
542
543
544
545
546
547
548
    "GteNewForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-multilingual-reranker-base",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
    ),
    "ModernBertForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-reranker-modernbert-base"
    ),
549
550
551
    "ModernBertForTokenClassification": _HfExamplesInfo(
        "disham993/electrical-ner-ModernBERT-base"
    ),
552
553
554
555
    "RobertaForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/quora-roberta-base"
    ),
    "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"),
556
557
}

558
559
_AUTOMATIC_CONVERTED_MODELS = {
    # Use as_seq_cls_model for automatic conversion
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
    "GemmaForSequenceClassification": _HfExamplesInfo(
        "BAAI/bge-reranker-v2-gemma",
        hf_overrides={
            "architectures": ["GemmaForSequenceClassification"],
            "classifier_from_token": ["Yes"],
            "method": "no_post_processing",
        },
    ),
    "LlamaForSequenceClassification": _HfExamplesInfo(
        "Skywork/Skywork-Reward-V2-Llama-3.2-1B"
    ),
    "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"),
    "Qwen3ForSequenceClassification": _HfExamplesInfo(
        "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
    ),
575
576
}

577
578
_MULTIMODAL_EXAMPLE_MODELS = {
    # [Decoder-only]
579
    "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
580
    "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereForAI/aya-vision-8b"),
581
582
583
584
    "BeeForConditionalGeneration": _HfExamplesInfo(
        "Open-Bee/Bee-8B-RL",
        trust_remote_code=True,
    ),
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
    "Blip2ForConditionalGeneration": _HfExamplesInfo(
        "Salesforce/blip2-opt-2.7b",
        extras={"6b": "Salesforce/blip2-opt-6.7b"},
    ),
    "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),
    "Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
        "CohereLabs/command-a-vision-07-2025"
    ),
    "DeepseekVLV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-vl2-tiny",
        extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
        max_transformers_version="4.48",
        transformers_version_reason="HF model is not compatible.",
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
    ),
600
601
602
    "DeepseekOCRForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-OCR",
    ),
603
604
605
    "DotsOCRForCausalLM": _HfExamplesInfo(
        "rednote-hilab/dots.ocr", trust_remote_code=True
    ),
606
    "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
607
608
609
610
    "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
        "baidu/ERNIE-4.5-VL-28B-A3B-PT",
        trust_remote_code=True,
    ),
611
    "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
612
    "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
613
    "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
614
615
616
617
618
619
620
621
622
    "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
        "ibm-granite/granite-speech-3.3-2b"
    ),
    "GLM4VForCausalLM": _HfExamplesInfo(
        "zai-org/glm-4v-9b",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
    ),
    "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
623
    "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
624
625
626
627
628
629
630
631
632
633
634
    "H2OVLChatModel": _HfExamplesInfo(
        "h2oai/h2ovl-mississippi-800m",
        trust_remote_code=True,
        extras={"2b": "h2oai/h2ovl-mississippi-2b"},
        max_transformers_version="4.48",
        transformers_version_reason="HF model is not compatible.",
    ),
    "HCXVisionForCausalLM": _HfExamplesInfo(
        "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
        trust_remote_code=True,
    ),
635
636
637
638
    "HunYuanVLForConditionalGeneration": _HfExamplesInfo(
        "tencent/HunyuanOCR",
        is_available_online=False,
    ),
639
640
    "Idefics3ForConditionalGeneration": _HfExamplesInfo(
        "HuggingFaceM4/Idefics3-8B-Llama3",
641
        extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
    ),
    "InternS1ForConditionalGeneration": _HfExamplesInfo(
        "internlm/Intern-S1", trust_remote_code=True
    ),
    "InternVLChatModel": _HfExamplesInfo(
        "OpenGVLab/InternVL2-1B",
        extras={
            "2B": "OpenGVLab/InternVL2-2B",
            "3.0": "OpenGVLab/InternVL3-1B",
            "3.5-qwen3": "OpenGVLab/InternVL3_5-1B",
            "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B",
            "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
        },
        trust_remote_code=True,
    ),
    "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"),
    "KeyeForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-8B-Preview",
        trust_remote_code=True,
    ),
    "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-1_5-8B",
        trust_remote_code=True,
    ),
    "KimiVLForConditionalGeneration": _HfExamplesInfo(
        "moonshotai/Kimi-VL-A3B-Instruct",
        extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
        trust_remote_code=True,
670
671
672
673
        max_transformers_version="4.53.3",
        transformers_version_reason="HF model uses deprecated transformers API "
        "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: "
        "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31",
674
    ),
675
676
677
678
    "LightOnOCRForConditionalGeneration": _HfExamplesInfo(
        "lightonai/LightOnOCR-1B",
        is_available_online=False,
    ),
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
    "Llama4ForConditionalGeneration": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
        max_model_len=10240,
        extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"},
    ),
    "LlavaForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-1.5-7b-hf",
        extras={
            "mistral": "mistral-community/pixtral-12b",
            "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic",
        },
    ),
    "LlavaNextForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-v1.6-mistral-7b-hf"
    ),
    "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/LLaVA-NeXT-Video-7B-hf"
    ),
    "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
    ),
    "MantisForConditionalGeneration": _HfExamplesInfo(
        "TIGER-Lab/Mantis-8B-siglip-llama3",
        max_transformers_version="4.48",
        transformers_version_reason="HF model is not compatible.",
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
    ),
    "MiDashengLMModel": _HfExamplesInfo(
        "mispeech/midashenglm-7b", trust_remote_code=True
    ),
    "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6", trust_remote_code=True),
    "MiniCPMV": _HfExamplesInfo(
        "openbmb/MiniCPM-Llama3-V-2_5",
        extras={
            "2.6": "openbmb/MiniCPM-V-2_6",
            "4.0": "openbmb/MiniCPM-V-4",
            "4.5": "openbmb/MiniCPM-V-4_5",
        },
        trust_remote_code=True,
    ),
    "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-VL-01",
        trust_remote_code=True,
    ),
    "Mistral3ForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
        extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
    ),
    "MolmoForCausalLM": _HfExamplesInfo(
        "allenai/Molmo-7B-D-0924",
        max_transformers_version="4.48",
        transformers_version_reason="Incorrectly-detected `tensorflow` import.",
        extras={"olmo": "allenai/Molmo-7B-O-0924"},
        trust_remote_code=True,
    ),
    "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
    "Llama_Nemotron_Nano_VL": _HfExamplesInfo(
        "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
        trust_remote_code=True,
    ),
    "NemotronH_Nano_VL_V2": _HfExamplesInfo(
        "nano_vl_dummy", is_available_online=False, trust_remote_code=True
    ),
Zero's avatar
Zero committed
742
743
744
    "OpenCUAForConditionalGeneration": _HfExamplesInfo(
        "xlangai/OpenCUA-7B", trust_remote_code=True
    ),
745
746
747
748
749
750
751
752
753
754
755
    "Ovis": _HfExamplesInfo(
        "AIDC-AI/Ovis2-1B",
        trust_remote_code=True,
        max_transformers_version="4.53",
        transformers_version_reason="HF model is not compatible",
        extras={
            "1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
            "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
        },
    ),
    "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
756
757
758
759
    "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
        "PaddlePaddle/PaddleOCR-VL",
        trust_remote_code=True,
    ),
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
    "PaliGemmaForConditionalGeneration": _HfExamplesInfo(
        "google/paligemma-3b-mix-224",
        extras={"v2": "google/paligemma2-3b-ft-docci-448"},
    ),
    "Phi3VForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3-vision-128k-instruct",
        trust_remote_code=True,
        max_transformers_version="4.48",
        transformers_version_reason="Use of deprecated imports which have been removed.",  # noqa: E501
        extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
    ),
    "Phi4MMForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
    ),
    "PixtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Pixtral-12B-2409",
776
777
778
779
        extras={
            "mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
            "ministral-3": "mistralai/Ministral-3-3B-Instruct-2512",
        },
780
        tokenizer_mode="mistral",
781
782
        # TODO: revert once Mistral-Large-3 and Ministral-3 are publicly available.
        is_available_online=False,
783
784
785
786
787
    ),
    "QwenVLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen-VL",
        extras={"chat": "Qwen/Qwen-VL-Chat"},
        trust_remote_code=True,
788
789
        max_transformers_version="4.53.3",
        transformers_version_reason="Use of deprecated imports which have been removed.",  # noqa: E501
790
791
792
793
794
795
796
797
798
799
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
    ),
    "Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2-Audio-7B-Instruct"
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"),
    "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-3B-Instruct",
        max_model_len=4096,
    ),
800
    "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
801
802
803
804
805
806
807
808
809
810
811
    "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"),
    "Qwen3VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-4B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
    "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
812
813
814
815
816
    "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
817
818
819
820
821
    "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
    "SkyworkR1VChatModel": _HfExamplesInfo(
        "Skywork/Skywork-R1V-38B", trust_remote_code=True
    ),
    "SmolVLMForConditionalGeneration": _HfExamplesInfo(
822
        "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
823
824
825
826
827
828
829
830
831
832
833
    ),
    "Step3VLForConditionalGeneration": _HfExamplesInfo(
        "stepfun-ai/step3", trust_remote_code=True
    ),
    "UltravoxModel": _HfExamplesInfo(
        "fixie-ai/ultravox-v0_5-llama-3_2-1b",
        trust_remote_code=True,
    ),
    "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
    "Tarsier2ForConditionalGeneration": _HfExamplesInfo(
        "omni-research/Tarsier2-Recap-7b",
834
835
836
837
        hf_overrides={
            "architectures": ["Tarsier2ForConditionalGeneration"],
            "model_type": "tarsier2",
        },
838
    ),
839
840
841
842
843
    "VoxtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Voxtral-Mini-3B-2507",
        # disable this temporarily until we support HF format
        is_available_online=False,
    ),
844
    # [Encoder-decoder]
845
    "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"),
846
    # [Cross-encoder]
847
    "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"),
848
849
}

850

851
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
852
853
854
    "MedusaModel": _HfExamplesInfo(
        "JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
    ),
855
856
    # Temporarily disabled.
    # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
    # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
    #     "JackFram/llama-160m",
    #     speculative_model="ibm-ai-platform/llama-160m-accelerator"
    # ),
    "DeepSeekMTPModel": _HfExamplesInfo(
        "luccafong/deepseek_mtp_main_random",
        speculative_model="luccafong/deepseek_mtp_draft_random",
        trust_remote_code=True,
    ),
    "EagleDeepSeekMTPModel": _HfExamplesInfo(
        "eagle618/deepseek-v3-random",
        speculative_model="eagle618/eagle-deepseek-v3-random",
        trust_remote_code=True,
    ),
    "EagleLlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Meta-Llama-3-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B",
        tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
    ),
    "Eagle3LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.1-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
        tokenizer="meta-llama/Llama-3.1-8B-Instruct",
        use_original_num_layers=True,
        max_model_len=10240,
    ),
885
886
887
888
889
    "EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
        "mistralai/Mistral-Large-3-675B-Instruct-2512",
        speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
        is_available_online=False,
    ),
890
891
892
893
894
895
896
    "LlamaForCausalLMEagle3": _HfExamplesInfo(
        "Qwen/Qwen3-8B",
        trust_remote_code=True,
        speculative_model="AngelSlim/Qwen3-8B_eagle3",
        tokenizer="Qwen/Qwen3-8B",
        use_original_num_layers=True,
    ),
zhiweiz's avatar
zhiweiz committed
897
898
899
900
    "EagleLlama4ForCausalLM": _HfExamplesInfo(
        "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
        trust_remote_code=True,
        speculative_model="morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
901
902
903
904
905
906
        tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "EagleMiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-1B-sft-bf16",
        trust_remote_code=True,
        speculative_model="openbmb/MiniCPM-2B-sft-bf16",
907
        speculative_method="eagle",
908
909
910
911
912
913
914
915
916
917
918
        tokenizer="openbmb/MiniCPM-2B-sft-bf16",
    ),
    "ErnieMTPModel": _HfExamplesInfo(
        "baidu/ERNIE-4.5-21B-A3B-PT",
        trust_remote_code=True,
        speculative_model="baidu/ERNIE-4.5-21B-A3B-PT",
    ),
    "Glm4MoeMTPModel": _HfExamplesInfo(
        "zai-org/GLM-4.5",
        speculative_model="zai-org/GLM-4.5",
    ),
XuruiYang's avatar
XuruiYang committed
919
920
921
    "LongCatFlashMTPModel": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat",
        trust_remote_code=True,
922
923
924
925
926
927
928
        speculative_model="meituan-longcat/LongCat-Flash-Chat",
    ),
    "MiMoMTPModel": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-7B-RL",
        trust_remote_code=True,
        speculative_model="XiaomiMiMo/MiMo-7B-RL",
    ),
929
930
    "Eagle3Qwen2_5vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-7B-Instruct",
931
932
        speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl",
    ),
933
934
935
936
    "Eagle3Qwen3vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-VL-8B-Instruct",
        speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
    ),
937
938
939
    "Qwen3NextMTP": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
    ),
940
941
}

942
_TRANSFORMERS_BACKEND_MODELS = {
943
    "TransformersEmbeddingModel": _HfExamplesInfo(
944
        "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0.dev"
945
946
947
    ),
    "TransformersForSequenceClassification": _HfExamplesInfo(
        "papluca/xlm-roberta-base-language-detection",
948
        min_transformers_version="5.0.0.dev",
949
950
951
952
    ),
    "TransformersForCausalLM": _HfExamplesInfo(
        "hmellor/Ilama-3.2-1B", trust_remote_code=True
    ),
953
    "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
954
    "TransformersMoEForCausalLM": _HfExamplesInfo(
955
        "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0.dev"
956
    ),
957
    "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
958
        "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0.dev"
959
960
    ),
    "TransformersMoEEmbeddingModel": _HfExamplesInfo(
961
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0.dev"
962
963
    ),
    "TransformersMoEForSequenceClassification": _HfExamplesInfo(
964
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0.dev"
965
    ),
966
967
968
969
    "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
    "TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
        "google/gemma-3-4b-it"
    ),
970
971
}

972
973
974
_EXAMPLE_MODELS = {
    **_TEXT_GENERATION_EXAMPLE_MODELS,
    **_EMBEDDING_EXAMPLE_MODELS,
975
    **_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS,
976
977
    **_MULTIMODAL_EXAMPLE_MODELS,
    **_SPECULATIVE_DECODING_EXAMPLE_MODELS,
978
    **_TRANSFORMERS_BACKEND_MODELS,
979
980
981
982
983
984
985
986
987
}


class HfExampleModels:
    def __init__(self, hf_models: Mapping[str, _HfExamplesInfo]) -> None:
        super().__init__()

        self.hf_models = hf_models

988
    def get_supported_archs(self) -> Set[str]:
989
990
991
        return self.hf_models.keys()

    def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
992
993
994
        try:
            return self.hf_models[model_arch]
        except KeyError:
995
996
997
            raise ValueError(
                f"No example model defined for {model_arch}; please update this file."
            ) from None
998

999
1000
1001
1002
1003
    def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
        for info in self.hf_models.values():
            if info.default == model_id:
                return info

1004
1005
1006
1007
1008
        # Fallback to extras
        for info in self.hf_models.values():
            if any(extra == model_id for extra in info.extras.values()):
                return info

1009
1010
1011
        raise ValueError(
            f"No example model defined for {model_id}; please update this file."
        )
1012

1013

Patrick von Platen's avatar
Patrick von Platen committed
1014
HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
1015
AUTO_EXAMPLE_MODELS = HfExampleModels(_AUTOMATIC_CONVERTED_MODELS)