registry.py 41.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Mapping, Set
5
from dataclasses import dataclass, field
6
from typing import Any, Literal
7
8
9
10

import pytest
from packaging.version import Version
from transformers import __version__ as TRANSFORMERS_VERSION
11

12
from vllm.config.model import ModelDType, TokenizerMode
13

14
15
16
17
18
19
20
21
22

@dataclass(frozen=True)
class _HfExamplesInfo:
    default: str
    """The default model to use for testing this architecture."""

    extras: Mapping[str, str] = field(default_factory=dict)
    """Extra models to use for testing this architecture."""

23
    tokenizer: str | None = None
24
25
    """Set the tokenizer to load for this architecture."""

26
    tokenizer_mode: TokenizerMode | str = "auto"
27
28
    """Set the tokenizer type for this architecture."""

29
    speculative_model: str | None = None
30
31
32
33
34
    """
    The default model to use for testing this architecture, which is only used
    for speculative decoding.
    """

35
36
37
38
39
    speculative_method: str | None = None
    """
    The method to use for speculative decoding.
    """

40
    min_transformers_version: str | None = None
41
42
43
44
    """
    The minimum version of HF Transformers that is required to run this model.
    """

45
    max_transformers_version: str | None = None
46
47
48
49
    """
    The maximum version of HF Transformers that this model runs on.
    """

50
    transformers_version_reason: dict[Literal["vllm", "hf"], str] | None = None
51
    """
52
53
54
    The type and reason to skip test for the minimum/maximum version requirement.
    vllm: skip all vLLM tests if the version requirement is not met.
    hf: only skip tests that uses HF runner if the version requirement is not met.
55
56
    """

57
    require_embed_inputs: bool = False
58
    """
59
60
    If `True`, enables prompt and multi-modal embedding inputs while
    disabling tokenization.
61
62
63
64
65
66
67
68
69
70
71
72
73
74
    """

    dtype: ModelDType = "auto"
    """
    The data type for the model weights and activations.
    """

    enforce_eager: bool = False
    """
    Whether to enforce eager execution. If True, we will
    disable CUDA graph and always execute the model in eager mode.
    If False, we will use CUDA graph and eager execution in hybrid.
    """

75
76
    is_available_online: bool = True
    """
77
    Set this to `False` if the name of this architecture no longer exists on
78
79
80
81
82
83
    the HF repo. To maintain backwards compatibility, we have not removed them
    from the main model registry, so without this flag the registry tests will
    fail.
    """

    trust_remote_code: bool = False
84
    """The `trust_remote_code` level required to load the model."""
85

86
    hf_overrides: dict[str, Any] = field(default_factory=dict)
87
    """The `hf_overrides` required to load the model."""
88

89
    max_model_len: int | None = None
90
91
92
93
94
    """
    The maximum model length to use for this model. Some models default to a
    length that is too large to fit into memory in CI.
    """

95
    revision: str | None = None
96
97
98
99
100
    """
    The specific revision (commit hash, tag, or branch) to use for the model.
    If not specified, the default revision will be used.
    """

101
    max_num_seqs: int | None = None
102
103
    """Maximum number of sequences to be processed in a single iteration."""

104
105
106
107
108
109
    use_original_num_layers: bool = False
    """
    If True, use the original number of layers from the model config 
    instead of minimal layers for testing.
    """

110
111
112
    def check_transformers_version(
        self,
        *,
113
        on_fail: Literal["error", "skip", "return"],
114
        check_version_reason: Literal["vllm", "hf"] = "hf",
115
116
        check_min_version: bool = True,
        check_max_version: bool = True,
117
    ) -> str | None:
118
119
120
121
        """
        If the installed transformers version does not meet the requirements,
        perform the given action.
        """
122
123
124
125
        if (
            self.min_transformers_version is None
            and self.max_transformers_version is None
        ):
126
            return None
127
128

        current_version = TRANSFORMERS_VERSION
129
        cur_base_version = Version(current_version).base_version
130
131
132
        min_version = self.min_transformers_version
        max_version = self.max_transformers_version
        msg = f"`transformers=={current_version}` installed, but `transformers"
133
134
        # Only check the base version for the min/max version, otherwise preview
        # models cannot be run because `x.yy.0.dev0`<`x.yy.0`
135
136
        if min_version and Version(cur_base_version) < Version(min_version):
            is_version_valid = not check_min_version
137
            msg += f">={min_version}` is required to run this model."
138
139
        elif max_version and Version(cur_base_version) > Version(max_version):
            is_version_valid = not check_max_version
140
141
            msg += f"<={max_version}` is required to run this model."
        else:
142
143
144
145
146
147
148
149
150
151
152
            is_version_valid = True

        # check if Transformers version breaks the corresponding model runner,
        # skip test when model runner not compatible
        is_reason_valid = not (
            check_version_reason
            and self.transformers_version_reason
            and check_version_reason in self.transformers_version_reason
        )
        is_transformers_valid = is_version_valid and is_reason_valid
        if is_transformers_valid:
153
            return None
154
155
156
        elif self.transformers_version_reason:
            for reason_type, reason in self.transformers_version_reason.items():
                msg += f" Reason({reason_type}): {reason}"
157
158
159

        if on_fail == "error":
            raise RuntimeError(msg)
160
        elif on_fail == "skip":
161
            pytest.skip(msg)
162

163
164
        return msg

165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    def check_available_online(
        self,
        *,
        on_fail: Literal["error", "skip"],
    ) -> None:
        """
        If the model is not available online, perform the given action.
        """
        if not self.is_available_online:
            msg = "Model is not available online"

            if on_fail == "error":
                raise RuntimeError(msg)
            else:
                pytest.skip(msg)

181
182
183

_TEXT_GENERATION_EXAMPLE_MODELS = {
    # [Decoder-only]
184
    "AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
185
    "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
186
187
    "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B", trust_remote_code=True),
    "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B", trust_remote_code=True),
188
    "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
    "ArcticForCausalLM": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-instruct", trust_remote_code=True
    ),
    "BaiChuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan-7B", trust_remote_code=True
    ),
    "BaichuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan2-7B-chat", trust_remote_code=True
    ),
    "BailingMoeForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-lite-1.5", trust_remote_code=True
    ),
    "BailingMoeV2ForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-mini-2.0", trust_remote_code=True
    ),
    "BambaForCausalLM": _HfExamplesInfo(
        "ibm-ai-platform/Bamba-9B-v1",
        extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"},
    ),
    "BloomForCausalLM": _HfExamplesInfo(
        "bigscience/bloom-560m", {"1b": "bigscience/bloomz-1b1"}
    ),
    "ChatGLMModel": _HfExamplesInfo(
        "zai-org/chatglm3-6b", trust_remote_code=True, max_transformers_version="4.48"
    ),
    "ChatGLMForConditionalGeneration": _HfExamplesInfo(
        "thu-coai/ShieldLM-6B-chatglm3",
        trust_remote_code=True,
    ),
    "CohereForCausalLM": _HfExamplesInfo(
219
        "CohereLabs/c4ai-command-r-v01", trust_remote_code=True
220
221
    ),
    "Cohere2ForCausalLM": _HfExamplesInfo(
222
        "CohereLabs/c4ai-command-r7b-12-2024",
223
224
        trust_remote_code=True,
    ),
225
    "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
226
227
228
229
    # FIXME: databricks/dbrx-instruct has been deleted
    "DbrxForCausalLM": _HfExamplesInfo(
        "databricks/dbrx-instruct", is_available_online=False
    ),
230
231
232
233
    "DeciLMForCausalLM": _HfExamplesInfo(
        "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
        trust_remote_code=True,
    ),
234
235
236
237
    "DeepseekForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-moe-16b-base",
        trust_remote_code=True,
    ),
238
239
240
241
242
243
244
245
    "DeepseekV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V2-Lite-Chat",
        trust_remote_code=True,
    ),
    "DeepseekV3ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V3",
        trust_remote_code=True,
    ),
246
    "DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
247
248
    "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"),
    "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"),
249
250
251
    "ExaoneForCausalLM": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
    ),
252
    "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
253
    "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
254
    "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
255
    "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
256
    "FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
257
    "GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
258
259
260
    "Gemma2ForCausalLM": _HfExamplesInfo(
        "google/gemma-2-9b", extras={"tiny": "google/gemma-2-2b-it"}
    ),
261
    "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
262
    "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
263
264
    "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
    "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
265
    "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
266
267
268
    "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
    "GPTBigCodeForCausalLM": _HfExamplesInfo(
        "bigcode/starcoder",
269
270
271
272
        extras={
            "tiny": "bigcode/tiny_starcoder_py",
            "santacoder": "bigcode/gpt_bigcode-santacoder",
        },
273
274
275
276
277
278
279
    ),
    "GPTJForCausalLM": _HfExamplesInfo(
        "Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"}
    ),
    "GPTNeoXForCausalLM": _HfExamplesInfo(
        "EleutherAI/pythia-70m", {"1b": "EleutherAI/pythia-1.4b"}
    ),
280
    "GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"),
281
282
    "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
    "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
283
    "GraniteMoeHybridForCausalLM": _HfExamplesInfo(
284
        "ibm-granite/granite-4.0-tiny-preview"
285
286
287
288
289
290
291
    ),
    "GraniteMoeSharedForCausalLM": _HfExamplesInfo(
        "ibm-research/moe-7b-1b-active-shared-experts"
    ),
    "Grok1ModelForCausalLM": _HfExamplesInfo(
        "hpcai-tech/grok-1", trust_remote_code=True
    ),
292
    "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
    "HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
        "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
    ),
    "InternLMForCausalLM": _HfExamplesInfo(
        "internlm/internlm-chat-7b", trust_remote_code=True
    ),
    "InternLM2ForCausalLM": _HfExamplesInfo(
        "internlm/internlm2-chat-7b", trust_remote_code=True
    ),
    "InternLM2VEForCausalLM": _HfExamplesInfo(
        "OpenGVLab/Mono-InternVL-2B", trust_remote_code=True
    ),
    "InternLM3ForCausalLM": _HfExamplesInfo(
        "internlm/internlm3-8b-instruct", trust_remote_code=True
    ),
308
    "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
309
310
311
    "Jais2ForCausalLM": _HfExamplesInfo(
        "inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
    ),
312
313
314
315
316
317
318
    "JambaForCausalLM": _HfExamplesInfo(
        "ai21labs/AI21-Jamba-1.5-Mini",
        extras={
            "tiny": "ai21labs/Jamba-tiny-dev",
            "random": "ai21labs/Jamba-tiny-random",
        },
    ),
319
320
321
    "KimiLinearForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True
    ),
322
    "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
Paul Pak's avatar
Paul Pak committed
323
324
325
    "Lfm2MoeForCausalLM": _HfExamplesInfo(
        "LiquidAI/LFM2-8B-A1B", min_transformers_version="4.58"
    ),
326
327
328
329
330
331
    "LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.2-1B-Instruct",
        extras={
            "guard": "meta-llama/Llama-Guard-3-1B",
            "hermes": "NousResearch/Hermes-3-Llama-3.1-8B",
            "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
332
            "tiny": "hmellor/tiny-random-LlamaForCausalLM",
333
334
335
336
337
338
339
340
341
342
343
        },
    ),
    "LLaMAForCausalLM": _HfExamplesInfo(
        "decapoda-research/llama-7b-hf", is_available_online=False
    ),
    "Llama4ForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "LongcatFlashForCausalLM": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
    ),
344
    "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
345
346
347
348
349
350
351
352
353
354
355
356
357
    "Mamba2ForCausalLM": _HfExamplesInfo(
        "mistralai/Mamba-Codestral-7B-v0.1",
        extras={
            "random": "yujiepan/mamba2-codestral-v0.1-tiny-random",
        },
    ),
    "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"),
    "MiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-2B-sft-bf16", trust_remote_code=True
    ),
    "MiniCPM3ForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM3-4B", trust_remote_code=True
    ),
358
    "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
359
360
361
362
363
364
365
366
    "MiniMaxText01ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-Text-01",
        trust_remote_code=True,
        revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3",
    ),
    "MiniMaxM1ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-M1-40k", trust_remote_code=True
    ),
367
    "MiniMaxM2ForCausalLM": _HfExamplesInfo(
youkaichao's avatar
youkaichao committed
368
369
        "MiniMaxAI/MiniMax-M2",
        trust_remote_code=True,
370
    ),
371
    "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
372
    "MistralLarge3ForCausalLM": _HfExamplesInfo(
373
        "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4"
374
    ),
375
376
377
378
    "MixtralForCausalLM": _HfExamplesInfo(
        "mistralai/Mixtral-8x7B-Instruct-v0.1",
        {"tiny": "TitanML/tiny-mixtral"},
    ),
379
    "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
380
381
    # FIXME: mosaicml/mpt-7b has been deleted
    "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b", is_available_online=False),
382
    "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
383
384
385
    "NemotronHForCausalLM": _HfExamplesInfo(
        "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
    ),
386
    "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
387
    "Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
388
    "Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
389
    "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
390
391
392
393
394
    "OpenPanguMTPModel": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
    ),
395
396
397
398
399
400
    "OPTForCausalLM": _HfExamplesInfo(
        "facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
    ),
    "OrionForCausalLM": _HfExamplesInfo(
        "OrionStarAI/Orion-14B-Chat", trust_remote_code=True
    ),
401
    "OuroForCausalLM": _HfExamplesInfo("ByteDance/Ouro-1.4B", trust_remote_code=True),
402
403
404
405
406
407
408
409
    "PanguEmbeddedForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Embedded-7B-V1.1", trust_remote_code=True
    ),
    "PanguUltraMoEForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
    ),
410
    "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
411
    "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
412
    "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
413
414
415
416
417
418
419
    "PhiMoEForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3.5-MoE-instruct", trust_remote_code=True
    ),
    "Plamo2ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-2-1b",
        trust_remote_code=True,
    ),
420
421
422
423
    "Plamo3ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-3-nict-2b-base",
        trust_remote_code=True,
    ),
424
425
426
    "QWenLMHeadModel": _HfExamplesInfo(
        "Qwen/Qwen-7B-Chat",
        max_transformers_version="4.53",
427
428
429
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
430
431
432
        trust_remote_code=True,
    ),
    "Qwen2ForCausalLM": _HfExamplesInfo(
433
434
435
436
437
        "Qwen/Qwen2-0.5B-Instruct",
        extras={
            "2.5": "Qwen/Qwen2.5-0.5B-Instruct",
            "2.5-1.5B": "Qwen/Qwen2.5-1.5B-Instruct",
        },
438
    ),
439
    "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
Jee Jee Li's avatar
Jee Jee Li committed
440
441
    "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
    "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
442
443
444
445
446
    "Qwen3NextForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct",
        extras={"tiny-random": "tiny-random/qwen3-next-moe"},
        min_transformers_version="4.56.3",
    ),
447
    "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
448
449
450
451
    "SeedOssForCausalLM": _HfExamplesInfo(
        "ByteDance-Seed/Seed-OSS-36B-Instruct",
        trust_remote_code=True,
    ),
452
    "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
453
    "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
454
    "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
455
    "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
456
457
458
459
    "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3", trust_remote_code=True),
    "SolarForCausalLM": _HfExamplesInfo(
        "upstage/solar-pro-preview-instruct", trust_remote_code=True
    ),
460
461
462
    "TeleChatForCausalLM": _HfExamplesInfo(
        "chuhac/TeleChat2-35B", trust_remote_code=True
    ),
463
464
465
466
467
468
469
470
471
472
473
    "TeleChat2ForCausalLM": _HfExamplesInfo(
        "Tele-AI/TeleChat2-3B", trust_remote_code=True
    ),
    "TeleFLMForCausalLM": _HfExamplesInfo(
        "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
    ),
    "XverseForCausalLM": _HfExamplesInfo(
        "xverse/XVERSE-7B-Chat",
        tokenizer="meta-llama/Llama-2-7b",
        trust_remote_code=True,
    ),
474
    "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
475
    "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
476
477
478
    "MiMoV2FlashForCausalLM": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-V2-Flash", trust_remote_code=True
    ),
479
    "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
480
481
482
483
}

_EMBEDDING_EXAMPLE_MODELS = {
    # [Text-only]
484
    "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
485
    "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
486
    "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
487
    "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
488
489
490
491
492
493
494
495
496
497
498
499
    "GteModel": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True
    ),
    "GteNewModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-base-en-v1.5",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewModel"]},
    ),
    "InternLM2ForRewardModel": _HfExamplesInfo(
        "internlm/internlm2-1_8b-reward", trust_remote_code=True
    ),
    "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"),
500
    "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
501
502
503
    "LlamaBidirectionalModel": _HfExamplesInfo(
        "nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True
    ),
504
    "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
505
506
507
508
509
510
    "ModernBertModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-modernbert-base", trust_remote_code=True
    ),
    "NomicBertModel": _HfExamplesInfo(
        "nomic-ai/nomic-embed-text-v2-moe", trust_remote_code=True
    ),
511
    "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
512
513
514
    "Qwen2ForRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-RM-72B",
        max_transformers_version="4.53",
515
516
517
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
518
519
520
521
    ),
    "Qwen2ForProcessRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-PRM-7B",
        max_transformers_version="4.53",
522
523
524
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
525
526
527
528
    ),
    "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"),
    "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"),
    "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"),
529
    "BertSpladeSparseEmbeddingModel": _HfExamplesInfo(
530
531
        "naver/splade-v3",
        hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
532
    ),
533
    # [Multimodal]
534
    "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
535
    "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
536
537
538
539
    "Phi3VForCausalLM": _HfExamplesInfo(
        "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
540
    "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"),
541
542
    "PrithviGeoSpatialMAE": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
543
        dtype="float16",
544
        enforce_eager=True,
545
546
        require_embed_inputs=True,
        # This is to avoid the model going OOM in CI
547
548
549
550
        max_num_seqs=32,
    ),
    "Terratorch": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
551
        dtype="float16",
552
        enforce_eager=True,
553
        require_embed_inputs=True,
554
555
556
        # This is to avoid the model going OOM in CI
        max_num_seqs=32,
    ),
557
558
}

559
560
_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
    # [Decoder-only]
561
562
563
    "GPT2ForSequenceClassification": _HfExamplesInfo(
        "nie3e/sentiment-polish-gpt2-small"
    ),
564
    # [Cross-encoder]
565
566
567
    "BertForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/ms-marco-MiniLM-L-6-v2"
    ),
568
    "BertForTokenClassification": _HfExamplesInfo("boltuix/NeuroBERT-NER"),
569
570
571
572
573
    "GteNewForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-multilingual-reranker-base",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
    ),
574
575
576
    "LlamaBidirectionalForSequenceClassification": _HfExamplesInfo(
        "nvidia/llama-nemotron-rerank-1b-v2", trust_remote_code=True
    ),
577
578
579
    "ModernBertForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-reranker-modernbert-base"
    ),
580
581
582
    "ModernBertForTokenClassification": _HfExamplesInfo(
        "disham993/electrical-ner-ModernBERT-base"
    ),
583
584
585
586
    "RobertaForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/quora-roberta-base"
    ),
    "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"),
587
588
}

589
590
_AUTOMATIC_CONVERTED_MODELS = {
    # Use as_seq_cls_model for automatic conversion
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
    "GemmaForSequenceClassification": _HfExamplesInfo(
        "BAAI/bge-reranker-v2-gemma",
        hf_overrides={
            "architectures": ["GemmaForSequenceClassification"],
            "classifier_from_token": ["Yes"],
            "method": "no_post_processing",
        },
    ),
    "LlamaForSequenceClassification": _HfExamplesInfo(
        "Skywork/Skywork-Reward-V2-Llama-3.2-1B"
    ),
    "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"),
    "Qwen3ForSequenceClassification": _HfExamplesInfo(
        "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
    ),
606
    "Qwen3ForTokenClassification": _HfExamplesInfo("bd2lcco/Qwen3-0.6B-finetuned"),
607
608
}

609
610
_MULTIMODAL_EXAMPLE_MODELS = {
    # [Decoder-only]
611
    "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
612
613
614
    "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
        "nvidia/audio-flamingo-3-hf", min_transformers_version="5.0.0.dev"
    ),
615
    "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
616
    "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
617
618
619
620
    "BeeForConditionalGeneration": _HfExamplesInfo(
        "Open-Bee/Bee-8B-RL",
        trust_remote_code=True,
    ),
621
622
623
624
625
626
627
628
629
630
631
632
    "Blip2ForConditionalGeneration": _HfExamplesInfo(
        "Salesforce/blip2-opt-2.7b",
        extras={"6b": "Salesforce/blip2-opt-6.7b"},
    ),
    "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),
    "Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
        "CohereLabs/command-a-vision-07-2025"
    ),
    "DeepseekVLV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-vl2-tiny",
        extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
        max_transformers_version="4.48",
633
        transformers_version_reason={"hf": "HF model is not compatible."},
634
635
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
    ),
636
637
638
    "DeepseekOCRForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-OCR",
    ),
639
640
641
    "DotsOCRForCausalLM": _HfExamplesInfo(
        "rednote-hilab/dots.ocr", trust_remote_code=True
    ),
642
    "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
643
644
645
646
    "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
        "baidu/ERNIE-4.5-VL-28B-A3B-PT",
        trust_remote_code=True,
    ),
647
    "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
648
    "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
649
    "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
650
651
652
653
654
655
656
657
658
    "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
        "ibm-granite/granite-speech-3.3-2b"
    ),
    "GLM4VForCausalLM": _HfExamplesInfo(
        "zai-org/glm-4v-9b",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
    ),
    "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
659
    "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
660
661
662
663
664
    "H2OVLChatModel": _HfExamplesInfo(
        "h2oai/h2ovl-mississippi-800m",
        trust_remote_code=True,
        extras={"2b": "h2oai/h2ovl-mississippi-2b"},
        max_transformers_version="4.48",
665
        transformers_version_reason={"hf": "HF model is not compatible."},
666
667
668
669
670
    ),
    "HCXVisionForCausalLM": _HfExamplesInfo(
        "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
        trust_remote_code=True,
    ),
671
672
    "HunYuanVLForConditionalGeneration": _HfExamplesInfo(
        "tencent/HunyuanOCR",
673
        hf_overrides={"num_experts": 0},
674
    ),
675
676
    "Idefics3ForConditionalGeneration": _HfExamplesInfo(
        "HuggingFaceM4/Idefics3-8B-Llama3",
677
        extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
678
    ),
oscardev256's avatar
oscardev256 committed
679
680
681
682
    "IsaacForConditionalGeneration": _HfExamplesInfo(
        "PerceptronAI/Isaac-0.1",
        trust_remote_code=True,
    ),
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
    "InternS1ForConditionalGeneration": _HfExamplesInfo(
        "internlm/Intern-S1", trust_remote_code=True
    ),
    "InternVLChatModel": _HfExamplesInfo(
        "OpenGVLab/InternVL2-1B",
        extras={
            "2B": "OpenGVLab/InternVL2-2B",
            "3.0": "OpenGVLab/InternVL3-1B",
            "3.5-qwen3": "OpenGVLab/InternVL3_5-1B",
            "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B",
            "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
        },
        trust_remote_code=True,
    ),
    "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"),
    "KeyeForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-8B-Preview",
        trust_remote_code=True,
    ),
    "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-1_5-8B",
        trust_remote_code=True,
    ),
    "KimiVLForConditionalGeneration": _HfExamplesInfo(
        "moonshotai/Kimi-VL-A3B-Instruct",
        extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
        trust_remote_code=True,
710
        max_transformers_version="4.53.3",
711
712
713
714
715
716
717
        transformers_version_reason={
            "hf": (
                "HF model uses deprecated transformers API "
                "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: "
                "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31"
            )
        },
718
    ),
719
    "LightOnOCRForConditionalGeneration": _HfExamplesInfo(
720
        "lightonai/LightOnOCR-1B-1025"
721
    ),
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
    "Llama4ForConditionalGeneration": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
        max_model_len=10240,
        extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"},
    ),
    "LlavaForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-1.5-7b-hf",
        extras={
            "mistral": "mistral-community/pixtral-12b",
            "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic",
        },
    ),
    "LlavaNextForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-v1.6-mistral-7b-hf"
    ),
    "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/LLaVA-NeXT-Video-7B-hf"
    ),
    "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
    ),
    "MantisForConditionalGeneration": _HfExamplesInfo(
        "TIGER-Lab/Mantis-8B-siglip-llama3",
        max_transformers_version="4.48",
746
        transformers_version_reason={"hf": "HF model is not compatible."},
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
    ),
    "MiDashengLMModel": _HfExamplesInfo(
        "mispeech/midashenglm-7b", trust_remote_code=True
    ),
    "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6", trust_remote_code=True),
    "MiniCPMV": _HfExamplesInfo(
        "openbmb/MiniCPM-Llama3-V-2_5",
        extras={
            "2.6": "openbmb/MiniCPM-V-2_6",
            "4.0": "openbmb/MiniCPM-V-4",
            "4.5": "openbmb/MiniCPM-V-4_5",
        },
        trust_remote_code=True,
    ),
    "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-VL-01",
        trust_remote_code=True,
    ),
    "Mistral3ForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
        extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
    ),
    "MolmoForCausalLM": _HfExamplesInfo(
        "allenai/Molmo-7B-D-0924",
        max_transformers_version="4.48",
773
774
775
        transformers_version_reason={
            "vllm": "Incorrectly-detected `tensorflow` import from processor."
        },
776
777
778
779
780
781
782
783
784
785
786
        extras={"olmo": "allenai/Molmo-7B-O-0924"},
        trust_remote_code=True,
    ),
    "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
    "Llama_Nemotron_Nano_VL": _HfExamplesInfo(
        "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
        trust_remote_code=True,
    ),
    "NemotronH_Nano_VL_V2": _HfExamplesInfo(
        "nano_vl_dummy", is_available_online=False, trust_remote_code=True
    ),
Zero's avatar
Zero committed
787
788
789
    "OpenCUAForConditionalGeneration": _HfExamplesInfo(
        "xlangai/OpenCUA-7B", trust_remote_code=True
    ),
790
791
792
793
    "Ovis": _HfExamplesInfo(
        "AIDC-AI/Ovis2-1B",
        trust_remote_code=True,
        max_transformers_version="4.53",
794
        transformers_version_reason={"hf": "HF model is not compatible"},
795
796
797
798
799
800
        extras={
            "1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
            "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
        },
    ),
    "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
801
802
803
804
    "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
        "PaddlePaddle/PaddleOCR-VL",
        trust_remote_code=True,
    ),
805
806
807
808
809
810
811
812
    "PaliGemmaForConditionalGeneration": _HfExamplesInfo(
        "google/paligemma-3b-mix-224",
        extras={"v2": "google/paligemma2-3b-ft-docci-448"},
    ),
    "Phi3VForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3-vision-128k-instruct",
        trust_remote_code=True,
        max_transformers_version="4.48",
813
814
815
        transformers_version_reason={
            "hf": "HF model use deprecated imports which have been removed."
        },  # noqa: E501
816
817
818
819
820
821
822
        extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
    ),
    "Phi4MMForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
    ),
    "PixtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Pixtral-12B-2409",
823
824
825
826
        extras={
            "mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
            "ministral-3": "mistralai/Ministral-3-3B-Instruct-2512",
        },
827
828
829
830
831
832
        tokenizer_mode="mistral",
    ),
    "QwenVLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen-VL",
        extras={"chat": "Qwen/Qwen-VL-Chat"},
        trust_remote_code=True,
833
        max_transformers_version="4.53.3",
834
835
836
        transformers_version_reason={
            "hf": "HF model uses deprecated imports which have been removed."
        },  # noqa: E501
837
838
839
840
841
842
843
844
845
846
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
    ),
    "Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2-Audio-7B-Instruct"
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"),
    "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-3B-Instruct",
        max_model_len=4096,
    ),
847
    "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
848
849
850
851
852
853
854
855
856
857
858
    "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"),
    "Qwen3VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-4B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
    "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
859
860
861
862
863
    "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
864
865
866
867
868
    "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
    "SkyworkR1VChatModel": _HfExamplesInfo(
        "Skywork/Skywork-R1V-38B", trust_remote_code=True
    ),
    "SmolVLMForConditionalGeneration": _HfExamplesInfo(
869
        "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
870
871
872
873
874
875
876
877
878
879
880
    ),
    "Step3VLForConditionalGeneration": _HfExamplesInfo(
        "stepfun-ai/step3", trust_remote_code=True
    ),
    "UltravoxModel": _HfExamplesInfo(
        "fixie-ai/ultravox-v0_5-llama-3_2-1b",
        trust_remote_code=True,
    ),
    "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
    "Tarsier2ForConditionalGeneration": _HfExamplesInfo(
        "omni-research/Tarsier2-Recap-7b",
881
882
883
884
        hf_overrides={
            "architectures": ["Tarsier2ForConditionalGeneration"],
            "model_type": "tarsier2",
        },
885
    ),
886
887
888
889
890
    "VoxtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Voxtral-Mini-3B-2507",
        # disable this temporarily until we support HF format
        is_available_online=False,
    ),
Patrick von Platen's avatar
Patrick von Platen committed
891
892
893
894
895
    "VoxtralStreamingGeneration": _HfExamplesInfo(
        "<place-holder>",
        # disable this temporarily until we support HF format
        is_available_online=False,
    ),
896
    # [Encoder-decoder]
897
898
899
900
    "WhisperForConditionalGeneration": _HfExamplesInfo(
        "openai/whisper-large-v3-turbo",
        extras={"v3": "openai/whisper-large-v3"},
    ),
901
    # [Cross-encoder]
902
    "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"),
903
904
}

905

906
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
907
908
909
    "MedusaModel": _HfExamplesInfo(
        "JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
    ),
910
911
    # Temporarily disabled.
    # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
    # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
    #     "JackFram/llama-160m",
    #     speculative_model="ibm-ai-platform/llama-160m-accelerator"
    # ),
    "DeepSeekMTPModel": _HfExamplesInfo(
        "luccafong/deepseek_mtp_main_random",
        speculative_model="luccafong/deepseek_mtp_draft_random",
        trust_remote_code=True,
    ),
    "EagleDeepSeekMTPModel": _HfExamplesInfo(
        "eagle618/deepseek-v3-random",
        speculative_model="eagle618/eagle-deepseek-v3-random",
        trust_remote_code=True,
    ),
    "EagleLlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Meta-Llama-3-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B",
        tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
    ),
    "Eagle3LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.1-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
        tokenizer="meta-llama/Llama-3.1-8B-Instruct",
        use_original_num_layers=True,
        max_model_len=10240,
    ),
940
941
942
    "EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
        "mistralai/Mistral-Large-3-675B-Instruct-2512",
        speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
943
        # TODO: revert once figuring out OOM in CI
944
945
        is_available_online=False,
    ),
946
947
948
949
950
951
952
    "LlamaForCausalLMEagle3": _HfExamplesInfo(
        "Qwen/Qwen3-8B",
        trust_remote_code=True,
        speculative_model="AngelSlim/Qwen3-8B_eagle3",
        tokenizer="Qwen/Qwen3-8B",
        use_original_num_layers=True,
    ),
zhiweiz's avatar
zhiweiz committed
953
954
955
956
    "EagleLlama4ForCausalLM": _HfExamplesInfo(
        "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
        trust_remote_code=True,
        speculative_model="morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
957
958
959
960
961
962
        tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "EagleMiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-1B-sft-bf16",
        trust_remote_code=True,
        speculative_model="openbmb/MiniCPM-2B-sft-bf16",
963
        speculative_method="eagle",
964
965
966
967
968
969
970
971
972
973
974
        tokenizer="openbmb/MiniCPM-2B-sft-bf16",
    ),
    "ErnieMTPModel": _HfExamplesInfo(
        "baidu/ERNIE-4.5-21B-A3B-PT",
        trust_remote_code=True,
        speculative_model="baidu/ERNIE-4.5-21B-A3B-PT",
    ),
    "Glm4MoeMTPModel": _HfExamplesInfo(
        "zai-org/GLM-4.5",
        speculative_model="zai-org/GLM-4.5",
    ),
XuruiYang's avatar
XuruiYang committed
975
976
977
    "LongCatFlashMTPModel": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat",
        trust_remote_code=True,
978
979
980
981
982
983
984
        speculative_model="meituan-longcat/LongCat-Flash-Chat",
    ),
    "MiMoMTPModel": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-7B-RL",
        trust_remote_code=True,
        speculative_model="XiaomiMiMo/MiMo-7B-RL",
    ),
985
986
    "Eagle3Qwen2_5vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-7B-Instruct",
987
988
        speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl",
    ),
989
990
991
992
    "Eagle3Qwen3vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-VL-8B-Instruct",
        speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
    ),
993
994
995
    "Qwen3NextMTP": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
    ),
996
997
}

998
_TRANSFORMERS_BACKEND_MODELS = {
999
    "TransformersEmbeddingModel": _HfExamplesInfo(
1000
        "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0.dev"
1001
1002
1003
    ),
    "TransformersForSequenceClassification": _HfExamplesInfo(
        "papluca/xlm-roberta-base-language-detection",
1004
        min_transformers_version="5.0.0.dev",
1005
1006
1007
1008
    ),
    "TransformersForCausalLM": _HfExamplesInfo(
        "hmellor/Ilama-3.2-1B", trust_remote_code=True
    ),
1009
    "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
1010
    "TransformersMoEForCausalLM": _HfExamplesInfo(
1011
        "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0.dev"
1012
    ),
1013
    "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
1014
        "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0.dev"
1015
1016
    ),
    "TransformersMoEEmbeddingModel": _HfExamplesInfo(
1017
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0.dev"
1018
1019
    ),
    "TransformersMoEForSequenceClassification": _HfExamplesInfo(
1020
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0.dev"
1021
    ),
1022
1023
1024
1025
    "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
    "TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
        "google/gemma-3-4b-it"
    ),
1026
1027
}

1028
1029
1030
_EXAMPLE_MODELS = {
    **_TEXT_GENERATION_EXAMPLE_MODELS,
    **_EMBEDDING_EXAMPLE_MODELS,
1031
    **_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS,
1032
1033
    **_MULTIMODAL_EXAMPLE_MODELS,
    **_SPECULATIVE_DECODING_EXAMPLE_MODELS,
1034
    **_TRANSFORMERS_BACKEND_MODELS,
1035
1036
1037
1038
1039
1040
1041
1042
1043
}


class HfExampleModels:
    def __init__(self, hf_models: Mapping[str, _HfExamplesInfo]) -> None:
        super().__init__()

        self.hf_models = hf_models

1044
    def get_supported_archs(self) -> Set[str]:
1045
1046
1047
        return self.hf_models.keys()

    def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
1048
1049
1050
        try:
            return self.hf_models[model_arch]
        except KeyError:
1051
1052
1053
            raise ValueError(
                f"No example model defined for {model_arch}; please update this file."
            ) from None
1054

1055
1056
1057
1058
1059
    def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
        for info in self.hf_models.values():
            if info.default == model_id:
                return info

1060
1061
1062
1063
1064
        # Fallback to extras
        for info in self.hf_models.values():
            if any(extra == model_id for extra in info.extras.values()):
                return info

1065
1066
1067
        raise ValueError(
            f"No example model defined for {model_id}; please update this file."
        )
1068

1069

Patrick von Platen's avatar
Patrick von Platen committed
1070
HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
1071
AUTO_EXAMPLE_MODELS = HfExampleModels(_AUTOMATIC_CONVERTED_MODELS)