registry.py 61.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Mapping, Set
5
from dataclasses import dataclass, field
6
from typing import Any, Literal
7
8
9

import pytest
from packaging.version import Version
10
from transformers import PretrainedConfig
11
from transformers import __version__ as TRANSFORMERS_VERSION
12

13
from vllm.config.model import ModelDType, TokenizerMode
14

15
16
17
18
19
20
21
22
23

@dataclass(frozen=True)
class _HfExamplesInfo:
    default: str
    """The default model to use for testing this architecture."""

    extras: Mapping[str, str] = field(default_factory=dict)
    """Extra models to use for testing this architecture."""

24
    tokenizer: str | None = None
25
26
    """Set the tokenizer to load for this architecture."""

27
    tokenizer_mode: TokenizerMode | str = "auto"
28
29
    """Set the tokenizer type for this architecture."""

30
    speculative_model: str | None = None
31
32
33
34
35
    """
    The default model to use for testing this architecture, which is only used
    for speculative decoding.
    """

36
37
38
39
40
    speculative_method: str | None = None
    """
    The method to use for speculative decoding.
    """

41
    min_transformers_version: str | None = None
42
43
44
45
    """
    The minimum version of HF Transformers that is required to run this model.
    """

46
    max_transformers_version: str | None = None
47
48
49
50
    """
    The maximum version of HF Transformers that this model runs on.
    """

51
    transformers_version_reason: dict[Literal["vllm", "hf"], str] | None = None
52
    """
53
54
55
    The type and reason to skip test for the minimum/maximum version requirement.
    vllm: skip all vLLM tests if the version requirement is not met.
    hf: only skip tests that uses HF runner if the version requirement is not met.
56
57
    """

58
    require_embed_inputs: bool = False
59
    """
60
61
    If `True`, enables prompt and multi-modal embedding inputs while
    disabling tokenization.
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    """

    dtype: ModelDType = "auto"
    """
    The data type for the model weights and activations.
    """

    enforce_eager: bool = False
    """
    Whether to enforce eager execution. If True, we will
    disable CUDA graph and always execute the model in eager mode.
    If False, we will use CUDA graph and eager execution in hybrid.
    """

76
77
78
79
80
81
    enable_prefix_caching: bool = True
    """
    Whether to enable prefix caching for the model. If True, we will test the model with
    prefix caching enabled. If False, we will test the model without prefix caching.
    """

82
83
    is_available_online: bool = True
    """
84
    Set this to `False` if the name of this architecture no longer exists on
85
86
87
88
89
90
    the HF repo. To maintain backwards compatibility, we have not removed them
    from the main model registry, so without this flag the registry tests will
    fail.
    """

    trust_remote_code: bool = False
91
    """The `trust_remote_code` level required to load the model."""
92

93
    hf_overrides: dict[str, Any] = field(default_factory=dict)
94
    """The `hf_overrides` required to load the model."""
95

96
    max_model_len: int | None = None
97
98
99
100
101
    """
    The maximum model length to use for this model. Some models default to a
    length that is too large to fit into memory in CI.
    """

102
103
104
105
106
    max_num_batched_tokens: int | None = None
    """
    The maximum number of tokens to be processed in a single batch.
    """

107
    revision: str | None = None
108
109
110
111
112
    """
    The specific revision (commit hash, tag, or branch) to use for the model.
    If not specified, the default revision will be used.
    """

113
    max_num_seqs: int | None = None
114
115
    """Maximum number of sequences to be processed in a single iteration."""

116
117
    use_original_num_layers: bool = False
    """
118
    If True, use the original number of layers from the model config
119
120
121
    instead of minimal layers for testing.
    """

122
123
124
    def check_transformers_version(
        self,
        *,
125
        on_fail: Literal["error", "skip", "return"],
126
        check_version_reason: Literal["vllm", "hf"] = "hf",
127
128
        check_min_version: bool = True,
        check_max_version: bool = True,
129
    ) -> str | None:
130
131
132
133
        """
        If the installed transformers version does not meet the requirements,
        perform the given action.
        """
134
135
136
137
        if (
            self.min_transformers_version is None
            and self.max_transformers_version is None
        ):
138
            return None
139
140

        current_version = TRANSFORMERS_VERSION
141
        cur_base_version = Version(current_version).base_version
142
143
144
        min_version = self.min_transformers_version
        max_version = self.max_transformers_version
        msg = f"`transformers=={current_version}` installed, but `transformers"
145
146
        # Only check the base version for the min/max version, otherwise preview
        # models cannot be run because `x.yy.0.dev0`<`x.yy.0`
147
148
        if min_version and Version(cur_base_version) < Version(min_version):
            is_version_valid = not check_min_version
149
            msg += f">={min_version}` is required to run this model."
150
151
        elif max_version and Version(cur_base_version) > Version(max_version):
            is_version_valid = not check_max_version
152
153
            msg += f"<={max_version}` is required to run this model."
        else:
154
155
156
157
158
159
160
161
162
163
164
            is_version_valid = True

        # check if Transformers version breaks the corresponding model runner,
        # skip test when model runner not compatible
        is_reason_valid = not (
            check_version_reason
            and self.transformers_version_reason
            and check_version_reason in self.transformers_version_reason
        )
        is_transformers_valid = is_version_valid and is_reason_valid
        if is_transformers_valid:
165
            return None
166
167
168
        elif self.transformers_version_reason:
            for reason_type, reason in self.transformers_version_reason.items():
                msg += f" Reason({reason_type}): {reason}"
169
170
171

        if on_fail == "error":
            raise RuntimeError(msg)
172
        elif on_fail == "skip":
173
            pytest.skip(msg)
174

175
176
        return msg

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
    def check_available_online(
        self,
        *,
        on_fail: Literal["error", "skip"],
    ) -> None:
        """
        If the model is not available online, perform the given action.
        """
        if not self.is_available_online:
            msg = "Model is not available online"

            if on_fail == "error":
                raise RuntimeError(msg)
            else:
                pytest.skip(msg)

193
194
195

_TEXT_GENERATION_EXAMPLE_MODELS = {
    # [Decoder-only]
196
    "AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
197
    "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
198
199
    "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B", trust_remote_code=True),
    "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B", trust_remote_code=True),
200
    "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
201
202
203
    "ArcticForCausalLM": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-instruct", trust_remote_code=True
    ),
204
    "AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
205
206
207
208
209
210
211
212
213
214
215
216
    "BaiChuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan-7B", trust_remote_code=True
    ),
    "BaichuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan2-7B-chat", trust_remote_code=True
    ),
    "BailingMoeForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-lite-1.5", trust_remote_code=True
    ),
    "BailingMoeV2ForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-mini-2.0", trust_remote_code=True
    ),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
217
218
219
    "BailingMoeV2_5ForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ring-2.5-1T", trust_remote_code=True
    ),
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
    "BambaForCausalLM": _HfExamplesInfo(
        "ibm-ai-platform/Bamba-9B-v1",
        extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"},
    ),
    "BloomForCausalLM": _HfExamplesInfo(
        "bigscience/bloom-560m", {"1b": "bigscience/bloomz-1b1"}
    ),
    "ChatGLMModel": _HfExamplesInfo(
        "zai-org/chatglm3-6b", trust_remote_code=True, max_transformers_version="4.48"
    ),
    "ChatGLMForConditionalGeneration": _HfExamplesInfo(
        "thu-coai/ShieldLM-6B-chatglm3",
        trust_remote_code=True,
    ),
    "CohereForCausalLM": _HfExamplesInfo(
235
        "CohereLabs/c4ai-command-r-v01", trust_remote_code=True
236
237
    ),
    "Cohere2ForCausalLM": _HfExamplesInfo(
238
        "CohereLabs/c4ai-command-r7b-12-2024",
239
240
        trust_remote_code=True,
    ),
241
    "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
242
243
244
245
    # FIXME: databricks/dbrx-instruct has been deleted
    "DbrxForCausalLM": _HfExamplesInfo(
        "databricks/dbrx-instruct", is_available_online=False
    ),
246
247
248
249
    "DeciLMForCausalLM": _HfExamplesInfo(
        "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
        trust_remote_code=True,
    ),
250
251
252
253
    "DeepseekForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-moe-16b-base",
        trust_remote_code=True,
    ),
254
255
256
257
258
259
260
261
    "DeepseekV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V2-Lite-Chat",
        trust_remote_code=True,
    ),
    "DeepseekV3ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V3",
        trust_remote_code=True,
    ),
262
    "DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
263
264
    "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"),
    "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"),
265
266
267
    "ExaoneForCausalLM": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
    ),
268
    "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
Kyungmin Lee's avatar
Kyungmin Lee committed
269
    "ExaoneMoEForCausalLM": _HfExamplesInfo(
270
        "LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
Kyungmin Lee's avatar
Kyungmin Lee committed
271
    ),
272
    "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
273
    "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
274
    "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
275
    "FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
276
    "GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
277
278
279
    "Gemma2ForCausalLM": _HfExamplesInfo(
        "google/gemma-2-9b", extras={"tiny": "google/gemma-2-2b-it"}
    ),
280
    "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
281
282
283
284
    "Gemma4ForCausalLM": _HfExamplesInfo(
        "google/gemma-4-E2B-it",
        min_transformers_version="5.0.0",
    ),
285
    "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
286
287
    "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
    "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
288
    "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
289
290
    "Glm4MoeLiteForCausalLM": _HfExamplesInfo(
        "zai-org/GLM-4.7-Flash",
291
        min_transformers_version="5.0.0",
292
    ),
Jee Jee Li's avatar
Jee Jee Li committed
293
294
295
    "GlmMoeDsaForCausalLM": _HfExamplesInfo(
        "zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
    ),
296
297
298
    "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
    "GPTBigCodeForCausalLM": _HfExamplesInfo(
        "bigcode/starcoder",
299
300
301
302
        extras={
            "tiny": "bigcode/tiny_starcoder_py",
            "santacoder": "bigcode/gpt_bigcode-santacoder",
        },
303
304
305
306
307
308
309
    ),
    "GPTJForCausalLM": _HfExamplesInfo(
        "Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"}
    ),
    "GPTNeoXForCausalLM": _HfExamplesInfo(
        "EleutherAI/pythia-70m", {"1b": "EleutherAI/pythia-1.4b"}
    ),
310
    "GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"),
311
312
    "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
    "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
313
    "GraniteMoeHybridForCausalLM": _HfExamplesInfo(
314
        "ibm-granite/granite-4.0-tiny-preview"
315
316
317
318
319
320
321
    ),
    "GraniteMoeSharedForCausalLM": _HfExamplesInfo(
        "ibm-research/moe-7b-1b-active-shared-experts"
    ),
    "Grok1ModelForCausalLM": _HfExamplesInfo(
        "hpcai-tech/grok-1", trust_remote_code=True
    ),
Bijaya Dangol's avatar
Bijaya Dangol committed
322
    "Grok1ForCausalLM": _HfExamplesInfo("xai-org/grok-2", trust_remote_code=True),
323
    "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
324
325
326
    "HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
        "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
    ),
327
    "HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True),
328
    "HyperCLOVAXForCausalLM": _HfExamplesInfo(
329
        "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
330
331
        trust_remote_code=True,
    ),
332
333
334
335
336
337
338
    "InternLMForCausalLM": _HfExamplesInfo(
        "internlm/internlm-chat-7b", trust_remote_code=True
    ),
    "InternLM2ForCausalLM": _HfExamplesInfo(
        "internlm/internlm2-chat-7b", trust_remote_code=True
    ),
    "InternLM2VEForCausalLM": _HfExamplesInfo(
339
340
341
342
343
344
345
346
347
        "OpenGVLab/Mono-InternVL-2B",
        trust_remote_code=True,
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": (
                "Custom config cannot be loaded with Transformers "
                "v5 because `vision_config` is not always set"
            )
        },
348
349
350
351
    ),
    "InternLM3ForCausalLM": _HfExamplesInfo(
        "internlm/internlm3-8b-instruct", trust_remote_code=True
    ),
352
353
354
355
356
357
    "IQuestCoderForCausalLM": _HfExamplesInfo(
        "IQuestLab/IQuest-Coder-V1-40B-Instruct", trust_remote_code=True
    ),
    "IQuestLoopCoderForCausalLM": _HfExamplesInfo(
        "IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
    ),
358
    "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
359
360
361
    "Jais2ForCausalLM": _HfExamplesInfo(
        "inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
    ),
362
363
364
365
366
367
368
    "JambaForCausalLM": _HfExamplesInfo(
        "ai21labs/AI21-Jamba-1.5-Mini",
        extras={
            "tiny": "ai21labs/Jamba-tiny-dev",
            "random": "ai21labs/Jamba-tiny-random",
        },
    ),
369
370
371
    "KimiLinearForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True
    ),
372
    "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
Paul Pak's avatar
Paul Pak committed
373
    "Lfm2MoeForCausalLM": _HfExamplesInfo(
374
375
376
377
378
        "LiquidAI/LFM2-8B-A1B",
        min_transformers_version="5.0.0",
        use_original_num_layers=True,
        # Initialize at least one MoE layer
        hf_overrides={"num_hidden_layers": 4},
Paul Pak's avatar
Paul Pak committed
379
    ),
380
381
382
383
384
385
    "LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.2-1B-Instruct",
        extras={
            "guard": "meta-llama/Llama-Guard-3-1B",
            "hermes": "NousResearch/Hermes-3-Llama-3.1-8B",
            "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
386
            "tiny": "hmellor/tiny-random-LlamaForCausalLM",
387
388
389
390
391
392
393
394
395
396
397
        },
    ),
    "LLaMAForCausalLM": _HfExamplesInfo(
        "decapoda-research/llama-7b-hf", is_available_online=False
    ),
    "Llama4ForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "LongcatFlashForCausalLM": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
    ),
398
    "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
399
400
401
402
403
404
405
406
407
408
409
410
411
    "Mamba2ForCausalLM": _HfExamplesInfo(
        "mistralai/Mamba-Codestral-7B-v0.1",
        extras={
            "random": "yujiepan/mamba2-codestral-v0.1-tiny-random",
        },
    ),
    "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"),
    "MiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-2B-sft-bf16", trust_remote_code=True
    ),
    "MiniCPM3ForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM3-4B", trust_remote_code=True
    ),
412
413
414
    "MiniCPM4ForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM4.1-8B", trust_remote_code=True
    ),
415
    "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
416
417
418
419
420
421
422
423
    "MiniMaxText01ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-Text-01",
        trust_remote_code=True,
        revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3",
    ),
    "MiniMaxM1ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-M1-40k", trust_remote_code=True
    ),
424
    "MiniMaxM2ForCausalLM": _HfExamplesInfo(
youkaichao's avatar
youkaichao committed
425
426
        "MiniMaxAI/MiniMax-M2",
        trust_remote_code=True,
427
    ),
428
    "Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"),
429
    "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
430
    "MistralLarge3ForCausalLM": _HfExamplesInfo(
431
        "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4"
432
    ),
433
434
435
436
    "MixtralForCausalLM": _HfExamplesInfo(
        "mistralai/Mixtral-8x7B-Instruct-v0.1",
        {"tiny": "TitanML/tiny-mixtral"},
    ),
437
    "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
438
439
    # FIXME: mosaicml/mpt-7b has been deleted
    "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b", is_available_online=False),
440
    "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
441
442
443
    "NemotronHForCausalLM": _HfExamplesInfo(
        "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
    ),
444
445
446
447
448
    "NemotronHPuzzleForCausalLM": _HfExamplesInfo(
        "",
        trust_remote_code=True,
        is_available_online=False,
    ),
449
    "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
450
    "Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
451
    "Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
452
    "OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"),
453
    "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
454
455
456
457
458
459
    "OPTForCausalLM": _HfExamplesInfo(
        "facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
    ),
    "OrionForCausalLM": _HfExamplesInfo(
        "OrionStarAI/Orion-14B-Chat", trust_remote_code=True
    ),
460
    "OuroForCausalLM": _HfExamplesInfo("ByteDance/Ouro-1.4B", trust_remote_code=True),
461
462
463
    "PanguEmbeddedForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Embedded-7B-V1.1", trust_remote_code=True
    ),
464
465
466
467
468
    "PanguProMoEV2ForCausalLM": _HfExamplesInfo(
        "",
        trust_remote_code=True,
        is_available_online=False,
    ),
469
470
471
472
473
    "PanguUltraMoEForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
    ),
474
475
476
477
    "Param2MoEForCausalLM": _HfExamplesInfo(
        "bharatgenai/Param2-17B-A2.4B-Thinking",
        trust_remote_code=True,
    ),
478
    "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
479
    "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
480
    "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
481
482
483
484
485
486
    "PhiMoEForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3.5-MoE-instruct", trust_remote_code=True
    ),
    "Plamo2ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-2-1b",
        trust_remote_code=True,
487
488
489
490
491
492
493
        max_transformers_version="4.57",
        transformers_version_reason={
            "hf": (
                "Custom model code uses `_tied_weight_keys: list[str]` but "
                "Transformers v5 now expects `_tied_weight_keys: dict[str, str]`"
            )
        },
494
    ),
495
496
497
498
    "Plamo3ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-3-nict-2b-base",
        trust_remote_code=True,
    ),
499
500
501
    "QWenLMHeadModel": _HfExamplesInfo(
        "Qwen/Qwen-7B-Chat",
        max_transformers_version="4.53",
502
503
504
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
505
506
507
        trust_remote_code=True,
    ),
    "Qwen2ForCausalLM": _HfExamplesInfo(
508
509
510
511
512
        "Qwen/Qwen2-0.5B-Instruct",
        extras={
            "2.5": "Qwen/Qwen2.5-0.5B-Instruct",
            "2.5-1.5B": "Qwen/Qwen2.5-1.5B-Instruct",
        },
513
    ),
514
    "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
Jee Jee Li's avatar
Jee Jee Li committed
515
516
    "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
    "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
517
518
519
520
521
    "Qwen3NextForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct",
        extras={"tiny-random": "tiny-random/qwen3-next-moe"},
        min_transformers_version="4.56.3",
    ),
522
523
524
525
    "Rnj1ForCausalLM": _HfExamplesInfo(
        "EssentialAI/rnj-1-instruct",
        is_available_online=False,
    ),
526
    "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
527
528
529
530
531
532
533
534
535
536
537
    "SarvamMoEForCausalLM": _HfExamplesInfo(
        "sarvamai/sarvam-30b",
        trust_remote_code=True,
        max_model_len=4096,
        is_available_online=True,
    ),
    "SarvamMLAForCausalLM": _HfExamplesInfo(
        "sarvamai/sarvam-105b",
        trust_remote_code=True,
        max_model_len=4096,
        is_available_online=True,
538
539
540
541
542
543
544
        max_transformers_version="5.3",
        transformers_version_reason={
            "vllm": (
                "vllm upgraded transformers above v5.4 where "
                "validate_rope() no longer accepts ignore_keys param"
            )
        },
545
    ),
546
547
548
549
    "SeedOssForCausalLM": _HfExamplesInfo(
        "ByteDance-Seed/Seed-OSS-36B-Instruct",
        trust_remote_code=True,
    ),
550
551
552
553
    "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
    "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
    "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
    "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
Li Xie's avatar
Li Xie committed
554
555
556
    "Step1ForCausalLM": _HfExamplesInfo(
        "stepfun-ai/Step-Audio-EditX", trust_remote_code=True
    ),
csy0225's avatar
csy0225 committed
557
    "Step3p5ForCausalLM": _HfExamplesInfo(
558
559
560
        "stepfun-ai/Step-3.5-Flash",
        use_original_num_layers=True,
        # Initialize at least one MoE layer
561
        hf_overrides={"num_hidden_layers": 4},
csy0225's avatar
csy0225 committed
562
    ),
563
564
565
566
    "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3", trust_remote_code=True),
    "SolarForCausalLM": _HfExamplesInfo(
        "upstage/solar-pro-preview-instruct", trust_remote_code=True
    ),
567
568
569
    "TeleChatForCausalLM": _HfExamplesInfo(
        "chuhac/TeleChat2-35B", trust_remote_code=True
    ),
570
571
572
    "TeleChat2ForCausalLM": _HfExamplesInfo(
        "Tele-AI/TeleChat2-3B", trust_remote_code=True
    ),
573
574
575
    "TeleChat3ForCausalLM": _HfExamplesInfo(
        "Tele-AI/TeleChat3-36B-Thinking", trust_remote_code=True
    ),
576
577
578
579
580
581
582
    "TeleFLMForCausalLM": _HfExamplesInfo(
        "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
    ),
    "XverseForCausalLM": _HfExamplesInfo(
        "xverse/XVERSE-7B-Chat",
        tokenizer="meta-llama/Llama-2-7b",
        trust_remote_code=True,
583
584
585
586
587
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": "XVERSE tokenizer is incompatible with transformers v5 "
            "(add_prefix_space / prepend_scheme mismatch).",
        },
588
    ),
589
    "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
590
    "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
591
592
593
    "MiMoV2FlashForCausalLM": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-V2-Flash", trust_remote_code=True
    ),
594
    "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
595
596
597
598
}

_EMBEDDING_EXAMPLE_MODELS = {
    # [Text-only]
599
    "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
600
    "ErnieModel": _HfExamplesInfo("shibing624/text2vec-base-chinese-sentence"),
601
602
603
    "BertSpladeSparseEmbeddingModel": _HfExamplesInfo(
        "naver/splade-v3",
        hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
604
    ),
605
    "BgeM3EmbeddingModel": _HfExamplesInfo("BAAI/bge-m3"),
606
    "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
607
    "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
608
    "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
609
610
611
612
613
614
615
616
    "GteModel": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True
    ),
    "GteNewModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-base-en-v1.5",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewModel"]},
    ),
617
618
619
620
    "JinaEmbeddingsV5Model": _HfExamplesInfo(
        "jinaai/jina-embeddings-v5-text-small",
        trust_remote_code=True,
    ),
621
    "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
622
623
624
    "LlamaBidirectionalModel": _HfExamplesInfo(
        "nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True
    ),
625
    "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
626
627
628
629
630
631
    "ModernBertModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-modernbert-base", trust_remote_code=True
    ),
    "NomicBertModel": _HfExamplesInfo(
        "nomic-ai/nomic-embed-text-v2-moe", trust_remote_code=True
    ),
632
    "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
633
634
    "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"),
    "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"),
chengchengpei's avatar
chengchengpei committed
635
636
637
    "VoyageQwen3BidirectionalEmbedModel": _HfExamplesInfo(
        "voyageai/voyage-4-nano", trust_remote_code=True
    ),
638
    "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"),
639
    # [Multimodal]
640
    "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
641
642
643
    "LlamaNemotronVLModel": _HfExamplesInfo(
        "nvidia/llama-nemotron-embed-vl-1b-v2", trust_remote_code=True
    ),
644
    "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
645
646
647
648
    "Phi3VForCausalLM": _HfExamplesInfo(
        "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
649
    "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"),
650
651
    "PrithviGeoSpatialMAE": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
652
        dtype="float16",
653
        enforce_eager=True,
654
655
        require_embed_inputs=True,
        # This is to avoid the model going OOM in CI
656
657
658
659
        max_num_seqs=32,
    ),
    "Terratorch": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
660
        dtype="float16",
661
        enforce_eager=True,
662
        require_embed_inputs=True,
663
664
665
        # This is to avoid the model going OOM in CI
        max_num_seqs=32,
    ),
666
667
}

668
669
670
671
672
673
_LATE_INTERACTION_EXAMPLE_MODELS = {
    # [Text-only]
    "HF_ColBERT": _HfExamplesInfo("answerdotai/answerai-colbert-small-v1"),
    "ColBERTModernBertModel": _HfExamplesInfo(
        "lightonai/GTE-ModernColBERT-v1",
        hf_overrides={"architectures": ["ColBERTModernBertModel"]},
674
    ),
675
676
677
678
679
    "ColBERTJinaRobertaModel": _HfExamplesInfo(
        "jinaai/jina-colbert-v2",
        trust_remote_code=True,
        hf_overrides={"architectures": ["ColBERTJinaRobertaModel"]},
    ),
680
681
682
683
684
    "ColBERTLfm2Model": _HfExamplesInfo(
        "LiquidAI/LFM2-ColBERT-350M",
        trust_remote_code=True,
        hf_overrides={"architectures": ["ColBERTLfm2Model"]},
    ),
685
    "JinaForRanking": _HfExamplesInfo("jinaai/jina-reranker-v3"),
686
687
688
689
    # [Multimodal]
    "ColModernVBertForRetrieval": _HfExamplesInfo(
        "ModernVBERT/colmodernvbert-merged",
    ),
690
    "ColPaliForRetrieval": _HfExamplesInfo("vidore/colpali-v1.3-hf"),
691
692
693
694
695
696
    "ColQwen3": _HfExamplesInfo(
        "TomoroAI/tomoro-colqwen3-embed-4b", trust_remote_code=True
    ),
    "OpsColQwen3Model": _HfExamplesInfo(
        "OpenSearch-AI/Ops-Colqwen3-4B", trust_remote_code=True
    ),
697
698
699
700
701
    "ColQwen3_5": _HfExamplesInfo(
        "athrael-soju/colqwen3.5-4.5B-v3",
        trust_remote_code=True,
        max_model_len=4096,
    ),
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
    "Qwen3VLNemotronEmbedModel": _HfExamplesInfo(
        "nvidia/nemotron-colembed-vl-4b-v2",
    ),
}


_REWARD_EXAMPLE_MODELS = {
    "InternLM2ForRewardModel": _HfExamplesInfo(
        "internlm/internlm2-1_8b-reward", trust_remote_code=True
    ),
    "Qwen2ForRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-RM-72B",
        max_transformers_version="4.53",
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
    ),
    "Qwen2ForProcessRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-PRM-7B",
        max_transformers_version="4.53",
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
    ),
}

_TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
    "BertForTokenClassification": _HfExamplesInfo("boltuix/NeuroBERT-NER"),
730
731
732
    "ErnieForTokenClassification": _HfExamplesInfo(
        "gyr66/Ernie-3.0-base-chinese-finetuned-ner"
    ),
733
734
735
736
737
738
    "ModernBertForTokenClassification": _HfExamplesInfo(
        "disham993/electrical-ner-ModernBERT-base"
    ),
}

_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
739
740
741
    "BertForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/ms-marco-MiniLM-L-6-v2"
    ),
742
743
744
    "ErnieForSequenceClassification": _HfExamplesInfo(
        "Forrest20231206/ernie-3.0-base-zh-cls",
    ),
745
746
747
    "GPT2ForSequenceClassification": _HfExamplesInfo(
        "nie3e/sentiment-polish-gpt2-small"
    ),
748
749
750
751
752
    "GteNewForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-multilingual-reranker-base",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
    ),
753
    "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"),
754
755
756
    "LlamaBidirectionalForSequenceClassification": _HfExamplesInfo(
        "nvidia/llama-nemotron-rerank-1b-v2", trust_remote_code=True
    ),
757
758
759
    "LlamaNemotronVLForSequenceClassification": _HfExamplesInfo(
        "nvidia/llama-nemotron-rerank-vl-1b-v2", trust_remote_code=True
    ),
760
761
762
763
764
765
766
    "ModernBertForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-reranker-modernbert-base"
    ),
    "RobertaForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/quora-roberta-base"
    ),
    "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"),
767
768
}

769
770
_AUTOMATIC_CONVERTED_MODELS = {
    # Use as_seq_cls_model for automatic conversion
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
    "GemmaForSequenceClassification": _HfExamplesInfo(
        "BAAI/bge-reranker-v2-gemma",
        hf_overrides={
            "architectures": ["GemmaForSequenceClassification"],
            "classifier_from_token": ["Yes"],
            "method": "no_post_processing",
        },
    ),
    "LlamaForSequenceClassification": _HfExamplesInfo(
        "Skywork/Skywork-Reward-V2-Llama-3.2-1B"
    ),
    "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"),
    "Qwen3ForSequenceClassification": _HfExamplesInfo(
        "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
    ),
786
    "Qwen3ForTokenClassification": _HfExamplesInfo("bd2lcco/Qwen3-0.6B-finetuned"),
787
788
789
790
791
792
793
794
795
    "Qwen3VLForSequenceClassification": _HfExamplesInfo(
        "Qwen/Qwen3-VL-Reranker-2B",
        is_available_online=False,
        hf_overrides={
            "architectures": ["Qwen3VLForSequenceClassification"],
            "classifier_from_token": ["no", "yes"],
            "is_original_qwen3_reranker": True,
        },
    ),
796
797
}

798
799
_MULTIMODAL_EXAMPLE_MODELS = {
    # [Decoder-only]
800
    "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
801
    "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
802
803
804
805
806
        "nvidia/audio-flamingo-3-hf",
        min_transformers_version="5.3.0",
        transformers_version_reason={
            "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
        },
807
    ),
808
    "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
809
810
811
812
813
        "nvidia/music-flamingo-2601-hf",
        min_transformers_version="5.3.0",
        transformers_version_reason={
            "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
        },
814
    ),
815
    "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
816
    "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
817
818
819
820
    "BeeForConditionalGeneration": _HfExamplesInfo(
        "Open-Bee/Bee-8B-RL",
        trust_remote_code=True,
    ),
821
822
823
824
825
    "Blip2ForConditionalGeneration": _HfExamplesInfo(
        "Salesforce/blip2-opt-2.7b",
        extras={"6b": "Salesforce/blip2-opt-6.7b"},
    ),
    "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),
826
827
828
829
830
831
832
833
    "Cheers": _HfExamplesInfo(
        "ai9stars/Cheers",
        trust_remote_code=True,
    ),
    "CheersForConditionalGeneration": _HfExamplesInfo(
        "ai9stars/Cheers",
        trust_remote_code=True,
    ),
834
835
836
837
838
839
840
    "Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
        "CohereLabs/command-a-vision-07-2025"
    ),
    "DeepseekVLV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-vl2-tiny",
        extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
        max_transformers_version="4.48",
841
        transformers_version_reason={"hf": "HF model is not compatible."},
842
    ),
843
844
845
    "DeepseekOCRForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-OCR",
    ),
RED's avatar
RED committed
846
847
848
    "DeepseekOCR2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-OCR-2",
    ),
849
850
851
    "DotsOCRForCausalLM": _HfExamplesInfo(
        "rednote-hilab/dots.ocr", trust_remote_code=True
    ),
852
    "Eagle2_5_VLForConditionalGeneration": _HfExamplesInfo(
853
854
        "nvidia/Eagle2.5-8B",
        trust_remote_code=True,
855
    ),
856
    "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
857
858
859
    "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
        "baidu/ERNIE-4.5-VL-28B-A3B-PT",
        trust_remote_code=True,
860
        revision="refs/pr/17",
861
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
862
863
864
865
    "Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-4.5-33B",
        min_transformers_version="5.6.0",
    ),
866
867
    "FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
        "allendou/FireRedASR2-LLM-vllm",
868
869
870
871
872
873
        trust_remote_code=True,
        max_transformers_version="5.1",
        transformers_version_reason={
            "vllm": "Incompatible with transformers v5.2+ "
            "(dict object has no attribute '__name__').",
        },
874
    ),
875
876
    "FireRedLIDForConditionalGeneration": _HfExamplesInfo(
        "PatchyTisa/FireRedLID-vllm",
877
878
879
880
881
882
        trust_remote_code=True,
        max_transformers_version="5.1",
        transformers_version_reason={
            "vllm": "Incompatible with transformers v5.2+ "
            "(dict object has no attribute '__name__').",
        },
883
    ),
884
885
    "FunASRForConditionalGeneration": _HfExamplesInfo(
        "allendou/Fun-ASR-Nano-2512-vllm",
886
887
888
889
890
891
        trust_remote_code=True,
        max_transformers_version="5.1",
        transformers_version_reason={
            "vllm": "Incompatible with transformers v5.2+ "
            "(dict object has no attribute '__name__').",
        },
892
    ),
893
894
895
    "FunAudioChatForConditionalGeneration": _HfExamplesInfo(
        "funaudiochat", is_available_online=False
    ),
896
    "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
897
    "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
898
899
900
901
    "Gemma4ForConditionalGeneration": _HfExamplesInfo(
        "google/gemma-4-E2B-it",
        min_transformers_version="5.5.0",
    ),
902
    "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
903
904
    "GlmAsrForConditionalGeneration": _HfExamplesInfo(
        "zai-org/GLM-ASR-Nano-2512",
905
        min_transformers_version="5.0.0",
906
    ),
907
908
909
910
    "Granite4VisionForConditionalGeneration": _HfExamplesInfo(
        "ibm-granite/granite-vision-4.1-4b",
        is_available_online=False,
    ),
911
    "GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"),
912
    "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
913
914
        "ibm-granite/granite-speech-3.3-2b",
        extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"},
915
916
917
918
919
920
921
    ),
    "GLM4VForCausalLM": _HfExamplesInfo(
        "zai-org/glm-4v-9b",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
    ),
    "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
922
    "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
923
924
    "GlmOcrForConditionalGeneration": _HfExamplesInfo(
        "zai-org/GLM-OCR",
925
        min_transformers_version="5.1.0",
926
    ),
927
928
929
930
931
    "H2OVLChatModel": _HfExamplesInfo(
        "h2oai/h2ovl-mississippi-800m",
        trust_remote_code=True,
        extras={"2b": "h2oai/h2ovl-mississippi-2b"},
        max_transformers_version="4.48",
932
        transformers_version_reason={"hf": "HF model is not compatible."},
933
934
935
936
    ),
    "HCXVisionForCausalLM": _HfExamplesInfo(
        "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
        trust_remote_code=True,
937
938
939
940
941
942
943
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": (
                "Custom config cannot be loaded with Transformers "
                "v5 because `text_config` is not always set"
            )
        },
944
    ),
945
946
947
948
    "HCXVisionV2ForCausalLM": _HfExamplesInfo(
        "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
        trust_remote_code=True,
    ),
949
950
    "HunYuanVLForConditionalGeneration": _HfExamplesInfo(
        "tencent/HunyuanOCR",
951
        hf_overrides={"num_experts": 0},
952
    ),
953
954
    "Idefics3ForConditionalGeneration": _HfExamplesInfo(
        "HuggingFaceM4/Idefics3-8B-Llama3",
955
        extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
956
    ),
oscardev256's avatar
oscardev256 committed
957
958
959
    "IsaacForConditionalGeneration": _HfExamplesInfo(
        "PerceptronAI/Isaac-0.1",
        trust_remote_code=True,
960
        extras={"0.2-2B-Preview": "PerceptronAI/Isaac-0.2-2B-Preview"},
oscardev256's avatar
oscardev256 committed
961
    ),
962
    "InternS1ForConditionalGeneration": _HfExamplesInfo(
963
964
965
966
967
968
        "internlm/Intern-S1",
        trust_remote_code=True,
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": "Custom tokenizer code is not compatible with Transformers v5."
        },
969
    ),
zxy's avatar
zxy committed
970
971
972
973
    "InternS1ProForConditionalGeneration": _HfExamplesInfo(
        "internlm/Intern-S1-Pro",
        trust_remote_code=True,
    ),
974
975
976
977
978
979
980
981
982
983
984
985
    "InternVLChatModel": _HfExamplesInfo(
        "OpenGVLab/InternVL2-1B",
        extras={
            "2B": "OpenGVLab/InternVL2-2B",
            "3.0": "OpenGVLab/InternVL3-1B",
            "3.5-qwen3": "OpenGVLab/InternVL3_5-1B",
            "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B",
            "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
        },
        trust_remote_code=True,
    ),
    "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"),
986
987
988
989
    "KananaVForConditionalGeneration": _HfExamplesInfo(
        "kakaocorp/kanana-1.5-v-3b-instruct",
        trust_remote_code=True,
    ),
990
991
992
993
994
995
996
997
    "KeyeForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-8B-Preview",
        trust_remote_code=True,
    ),
    "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-1_5-8B",
        trust_remote_code=True,
    ),
998
999
1000
1001
1002
1003
1004
1005
1006
    "MoonshotKimiaForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-Audio-7B-Instruct",
        tokenizer_mode="kimi_audio",
        trust_remote_code=True,
    ),
    "KimiK25ForConditionalGeneration": _HfExamplesInfo(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True,
    ),
1007
1008
1009
1010
    "KimiVLForConditionalGeneration": _HfExamplesInfo(
        "moonshotai/Kimi-VL-A3B-Instruct",
        extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
        trust_remote_code=True,
1011
        max_transformers_version="4.53.3",
1012
1013
1014
1015
1016
1017
1018
        transformers_version_reason={
            "hf": (
                "HF model uses deprecated transformers API "
                "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: "
                "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31"
            )
        },
1019
    ),
1020
    "LightOnOCRForConditionalGeneration": _HfExamplesInfo(
1021
        "lightonai/LightOnOCR-1B-1025"
1022
    ),
1023
1024
1025
1026
    "Lfm2VlForConditionalGeneration": _HfExamplesInfo(
        "LiquidAI/LFM2-VL-450M",
        min_transformers_version="5.0.0",
    ),
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
    "Llama4ForConditionalGeneration": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
        max_model_len=10240,
        extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"},
    ),
    "LlavaForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-1.5-7b-hf",
        extras={
            "mistral": "mistral-community/pixtral-12b",
            "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic",
        },
    ),
    "LlavaNextForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-v1.6-mistral-7b-hf"
    ),
    "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/LLaVA-NeXT-Video-7B-hf"
    ),
    "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
    ),
    "MantisForConditionalGeneration": _HfExamplesInfo(
        "TIGER-Lab/Mantis-8B-siglip-llama3",
        max_transformers_version="4.48",
1051
        transformers_version_reason={"hf": "HF model is not compatible."},
1052
1053
1054
1055
1056
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
    ),
    "MiDashengLMModel": _HfExamplesInfo(
        "mispeech/midashenglm-7b", trust_remote_code=True
    ),
1057
1058
1059
1060
1061
1062
1063
1064
    "MiniCPMO": _HfExamplesInfo(
        "openbmb/MiniCPM-o-2_6",
        trust_remote_code=True,
        max_transformers_version="4.57",
        transformers_version_reason={
            "hf": "Custom processor code is not compatible with Transformers v5."
        },
    ),
1065
1066
1067
1068
1069
1070
1071
    "MiniCPMV": _HfExamplesInfo(
        "openbmb/MiniCPM-Llama3-V-2_5",
        extras={
            "2.6": "openbmb/MiniCPM-V-2_6",
            "4.0": "openbmb/MiniCPM-V-4",
            "4.5": "openbmb/MiniCPM-V-4_5",
        },
1072
1073
1074
1075
1076
1077
1078
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": (
                "MiniCPMVBatchFeature is incompatible with its base class in "
                "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
            )
        },
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
        trust_remote_code=True,
    ),
    "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-VL-01",
        trust_remote_code=True,
    ),
    "Mistral3ForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
        extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
    ),
    "MolmoForCausalLM": _HfExamplesInfo(
        "allenai/Molmo-7B-D-0924",
        max_transformers_version="4.48",
1092
1093
1094
        transformers_version_reason={
            "vllm": "Incorrectly-detected `tensorflow` import from processor."
        },
1095
1096
1097
        extras={"olmo": "allenai/Molmo-7B-O-0924"},
        trust_remote_code=True,
    ),
1098
1099
1100
1101
1102
1103
1104
1105
    "Molmo2ForConditionalGeneration": _HfExamplesInfo(
        "allenai/Molmo2-8B",
        extras={"olmo": "allenai/Molmo2-O-7B"},
        min_transformers_version="4.51",
        trust_remote_code=True,
        # required by current PrefixLM implementation
        max_num_batched_tokens=31872,
    ),
1106
1107
1108
1109
1110
1111
    "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
    "Llama_Nemotron_Nano_VL": _HfExamplesInfo(
        "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
        trust_remote_code=True,
    ),
    "NemotronH_Nano_VL_V2": _HfExamplesInfo(
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
        "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
        max_model_len=4096,
        # NemotronH layers are constructed via `hybrid_override_pattern`:
        use_original_num_layers=True,
        hf_overrides={
            "vision_config": PretrainedConfig(
                args={
                    "min_num_patches": 1,  # Trigger image dynamic res
                    "max_num_patches": 12,
                    "model": "vit_huge_patch16_224",
                },
                # Trigger conv3d:
                video_temporal_patch_size=2,
            ),
            "text_config": {
                "num_hidden_layers": 2,
                "hybrid_override_pattern": "M*",
            },
        },
        trust_remote_code=True,
1132
    ),
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
    # NemotronH_Nano_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2
    # Use the same registry test as NemotronH_Nano_VL_V2 above
    "NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo(
        "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
        max_model_len=4096,
        use_original_num_layers=True,
        hf_overrides={
            "vision_config": PretrainedConfig(
                args={
                    "min_num_patches": 1,
                    "max_num_patches": 12,
                    "model": "vit_huge_patch16_224",
                },
                video_temporal_patch_size=2,
            ),
            "text_config": {
                "num_hidden_layers": 2,
                "hybrid_override_pattern": "M*",
            },
        },
        trust_remote_code=True,
    ),
    # NemotronH_Super_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2 as well
    # Use the same registry test as NemotronH_Nano_VL_V2 above
    "NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo(
        "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
        max_model_len=4096,
        use_original_num_layers=True,
        hf_overrides={
            "vision_config": PretrainedConfig(
                args={
                    "min_num_patches": 1,
                    "max_num_patches": 12,
                    "model": "vit_huge_patch16_224",
                },
                video_temporal_patch_size=2,
            ),
            "text_config": {
                "num_hidden_layers": 2,
                "hybrid_override_pattern": "M*",
            },
        },
        trust_remote_code=True,
    ),
Zero's avatar
Zero committed
1177
    "OpenCUAForConditionalGeneration": _HfExamplesInfo(
1178
1179
1180
1181
1182
1183
        "xlangai/OpenCUA-7B",
        trust_remote_code=True,
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": "Tokenizer cannot be initialised in Transformers v5."
        },
Zero's avatar
Zero committed
1184
    ),
1185
1186
1187
1188
1189
    "OpenPanguVLForConditionalGeneration": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-VL-7B",
        trust_remote_code=True,
        max_model_len=4096,
        enforce_eager=True,
1190
1191
1192
1193
1194
1195
1196
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": (
                "OpenPanguVLVideoProcessorInitKwargs does not specify total=False, "
                "making all kwargs required. See https://huggingface.co/FreedomIntelligence/openPangu-VL-7B/discussions/2"
            )
        },
1197
    ),
1198
1199
1200
1201
    "Ovis": _HfExamplesInfo(
        "AIDC-AI/Ovis2-1B",
        trust_remote_code=True,
        max_transformers_version="4.53",
1202
        transformers_version_reason={"hf": "HF model is not compatible"},
1203
1204
1205
1206
1207
        extras={
            "1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
            "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
        },
    ),
1208
1209
1210
1211
1212
1213
1214
1215
    "Ovis2_5": _HfExamplesInfo(
        "AIDC-AI/Ovis2.5-2B",
        trust_remote_code=True,
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": "Custom processor code is not compatible with Transformers v5."
        },
    ),
1216
1217
1218
1219
    "Ovis2_6ForCausalLM": _HfExamplesInfo(
        "AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
    ),
    "Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
1220
1221
1222
1223
1224
1225
        "AIDC-AI/Ovis2.6-30B-A3B",
        trust_remote_code=True,
        max_transformers_version="4.57",
        transformers_version_reason={
            "vllm": "Custom processor code is not compatible with Transformers v5."
        },
1226
    ),
1227
1228
1229
1230
    "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
        "PaddlePaddle/PaddleOCR-VL",
        trust_remote_code=True,
    ),
1231
1232
1233
1234
1235
1236
1237
1238
    "PaliGemmaForConditionalGeneration": _HfExamplesInfo(
        "google/paligemma-3b-mix-224",
        extras={"v2": "google/paligemma2-3b-ft-docci-448"},
    ),
    "Phi3VForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3-vision-128k-instruct",
        trust_remote_code=True,
        max_transformers_version="4.48",
1239
1240
1241
        transformers_version_reason={
            "hf": "HF model use deprecated imports which have been removed."
        },  # noqa: E501
1242
1243
        extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
    ),
1244
    "Phi4ForCausalLMV": _HfExamplesInfo(
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
        "microsoft/Phi-4-reasoning-vision-15B",
        trust_remote_code=True,
        max_transformers_version="5.3",
        transformers_version_reason={
            "vllm": (
                "vllm upgraded transformers above v5.4 where HF model "
                "custom code uses siglip2 internals "
                "(filter_out_non_signature_kwargs) removed "
                "by huggingface/transformers#43514"
            )
        },
1256
    ),
1257
1258
1259
1260
1261
    "Phi4MMForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
    ),
    "PixtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Pixtral-12B-2409",
1262
1263
1264
1265
        extras={
            "mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
            "ministral-3": "mistralai/Ministral-3-3B-Instruct-2512",
        },
1266
1267
1268
1269
1270
1271
        tokenizer_mode="mistral",
    ),
    "QwenVLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen-VL",
        extras={"chat": "Qwen/Qwen-VL-Chat"},
        trust_remote_code=True,
1272
        max_transformers_version="4.53.3",
1273
1274
1275
        transformers_version_reason={
            "hf": "HF model uses deprecated imports which have been removed."
        },  # noqa: E501
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
    ),
    "Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2-Audio-7B-Instruct"
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"),
    "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-3B-Instruct",
        max_model_len=4096,
    ),
1286
    "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
    "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"),
    "Qwen3VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-4B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
    "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
1298
    "Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
1299
        "Qwen/Qwen3.5-0.8B",
1300
1301
1302
        max_model_len=4096,
    ),
    "Qwen3_5MoeForConditionalGeneration": _HfExamplesInfo(
1303
        "Qwen/Qwen3.5-35B-A3B",
1304
1305
        max_model_len=4096,
    ),
1306
1307
1308
1309
1310
    "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
Roger Wang's avatar
Roger Wang committed
1311
    "Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
1312
        "Qwen/Qwen3-ASR-0.6B",
Roger Wang's avatar
Roger Wang committed
1313
1314
1315
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
1316
    "Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
1317
        "Qwen/Qwen3-ASR-0.6B",
1318
1319
1320
1321
        max_model_len=4096,
        min_transformers_version="4.57",
        hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
    ),
1322
1323
1324
1325
1326
1327
    "Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
        "Qwen/Qwen3-ForcedAligner-0.6B",
        max_model_len=4096,
        min_transformers_version="4.57",
        hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
    ),
1328
1329
1330
1331
1332
    "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
    "SkyworkR1VChatModel": _HfExamplesInfo(
        "Skywork/Skywork-R1V-38B", trust_remote_code=True
    ),
    "SmolVLMForConditionalGeneration": _HfExamplesInfo(
1333
        "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
1334
1335
1336
1337
    ),
    "Step3VLForConditionalGeneration": _HfExamplesInfo(
        "stepfun-ai/step3", trust_remote_code=True
    ),
ltd0924's avatar
ltd0924 committed
1338
1339
1340
    "StepVLForConditionalGeneration": _HfExamplesInfo(
        "stepfun-ai/Step3-VL-10B", trust_remote_code=True
    ),
1341
1342
1343
1344
1345
1346
1347
    "UltravoxModel": _HfExamplesInfo(
        "fixie-ai/ultravox-v0_5-llama-3_2-1b",
        trust_remote_code=True,
    ),
    "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
    "Tarsier2ForConditionalGeneration": _HfExamplesInfo(
        "omni-research/Tarsier2-Recap-7b",
1348
1349
1350
1351
        hf_overrides={
            "architectures": ["Tarsier2ForConditionalGeneration"],
            "model_type": "tarsier2",
        },
1352
1353
1354
1355
1356
1357
1358
1359
        max_transformers_version="5.3",
        transformers_version_reason={
            "vllm": (
                "Qwen2VLConfig was split into Qwen2VLConfig + "
                "Qwen2VLTextConfig in transformers v5, breaking "
                "attribute access (num_attention_heads, hidden_size, etc.)"
            )
        },
1360
    ),
1361
1362
    "VoxtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Voxtral-Mini-3B-2507",
1363
        tokenizer_mode="mistral",
1364
    ),
1365
    "VoxtralRealtimeGeneration": _HfExamplesInfo(
1366
1367
1368
        "mistralai/Voxtral-Mini-4B-Realtime-2602",
        enforce_eager=True,
        tokenizer_mode="mistral",
Patrick von Platen's avatar
Patrick von Platen committed
1369
    ),
1370
    # [Encoder-decoder]
1371
1372
    "CohereAsrForConditionalGeneration": _HfExamplesInfo(
        "CohereLabs/cohere-transcribe-03-2026",
Ekagra Ranjan's avatar
Ekagra Ranjan committed
1373
1374
1375
        trust_remote_code=True,
        is_available_online=False,  # TODO (ekagra): revert after asr release
    ),
1376
1377
1378
    "NemotronParseForConditionalGeneration": _HfExamplesInfo(
        "nvidia/NVIDIA-Nemotron-Parse-v1.1", trust_remote_code=True
    ),
1379
1380
1381
1382
    "WhisperForConditionalGeneration": _HfExamplesInfo(
        "openai/whisper-large-v3-turbo",
        extras={"v3": "openai/whisper-large-v3"},
    ),
1383
    # [Cross-encoder]
1384
    "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"),
1385
1386
}

1387

1388
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1389
    # [Medusa]
1390
1391
1392
    "MedusaModel": _HfExamplesInfo(
        "JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
    ),
1393
1394
    # Temporarily disabled.
    # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
1395
1396
1397
1398
    # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
    #     "JackFram/llama-160m",
    #     speculative_model="ibm-ai-platform/llama-160m-accelerator"
    # ),
1399
1400
1401
1402
1403
1404
1405
1406
    # [DFlash]
    "DFlashDraftModel": _HfExamplesInfo(
        "Qwen/Qwen3.5-4B",
        speculative_model="z-lab/Qwen3.5-4B-DFlash",
        use_original_num_layers=True,  # Need all layers since DFlash has >1 layer,
        max_model_len=8192,  # Reduce max len to ensure test runs in low-VRAM CI env
        max_num_seqs=32,
    ),
1407
    # [Eagle]
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
    "EagleDeepSeekMTPModel": _HfExamplesInfo(
        "eagle618/deepseek-v3-random",
        speculative_model="eagle618/eagle-deepseek-v3-random",
        trust_remote_code=True,
    ),
    "EagleLlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Meta-Llama-3-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B",
        tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
    ),
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
    "Eagle3DeepseekV2ForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True,
        speculative_model="AQ-MedAI/Kimi-K25-eagle3",
        tokenizer="moonshotai/Kimi-K2.5",
    ),
    "Eagle3DeepseekV3ForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True,
        speculative_model="AQ-MedAI/Kimi-K25-eagle3",
        tokenizer="moonshotai/Kimi-K2.5",
    ),
1431
1432
1433
1434
1435
1436
1437
1438
    "Eagle3LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.1-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
        tokenizer="meta-llama/Llama-3.1-8B-Instruct",
        use_original_num_layers=True,
        max_model_len=10240,
    ),
1439
1440
1441
1442
1443
1444
    "Eagle3MiniMaxM2ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-M2",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
        tokenizer="MiniMaxAI/MiniMax-M2",
    ),
1445
1446
1447
    "EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
        "mistralai/Mistral-Large-3-675B-Instruct-2512",
        speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
1448
        # TODO: revert once figuring out OOM in CI
1449
1450
        is_available_online=False,
    ),
1451
1452
1453
1454
1455
1456
1457
    "LlamaForCausalLMEagle3": _HfExamplesInfo(
        "Qwen/Qwen3-8B",
        trust_remote_code=True,
        speculative_model="AngelSlim/Qwen3-8B_eagle3",
        tokenizer="Qwen/Qwen3-8B",
        use_original_num_layers=True,
    ),
zhiweiz's avatar
zhiweiz committed
1458
1459
1460
1461
    "EagleLlama4ForCausalLM": _HfExamplesInfo(
        "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
        trust_remote_code=True,
        speculative_model="morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
1462
1463
1464
1465
1466
1467
        tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "EagleMiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-1B-sft-bf16",
        trust_remote_code=True,
        speculative_model="openbmb/MiniCPM-2B-sft-bf16",
1468
        speculative_method="eagle",
1469
1470
        tokenizer="openbmb/MiniCPM-2B-sft-bf16",
    ),
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
    "Eagle3Qwen2_5vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-7B-Instruct",
        speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl",
    ),
    "Eagle3Qwen3vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-VL-8B-Instruct",
        speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
    ),
    # [MTP]
    "DeepSeekMTPModel": _HfExamplesInfo(
        "luccafong/deepseek_mtp_main_random",
        speculative_model="luccafong/deepseek_mtp_draft_random",
        trust_remote_code=True,
    ),
1485
1486
1487
1488
1489
    "ErnieMTPModel": _HfExamplesInfo(
        "baidu/ERNIE-4.5-21B-A3B-PT",
        trust_remote_code=True,
        speculative_model="baidu/ERNIE-4.5-21B-A3B-PT",
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
1490
1491
1492
    "ExaoneMoeMTP": _HfExamplesInfo(
        "LGAI-EXAONE/K-EXAONE-236B-A23B",
        speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
1493
        min_transformers_version="5.1.0",
1494
        enable_prefix_caching=False,
Kyungmin Lee's avatar
Kyungmin Lee committed
1495
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
1496
1497
1498
1499
1500
    "Exaone4_5_MTP": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-4.5-33B",
        speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
        min_transformers_version="5.6.0",
    ),
1501
1502
1503
1504
    "ExtractHiddenStatesModel": _HfExamplesInfo(
        "Qwen/Qwen3-8B",
        speculative_method="extract_hidden_states",
    ),
1505
1506
1507
1508
    "Glm4MoeMTPModel": _HfExamplesInfo(
        "zai-org/GLM-4.5",
        speculative_model="zai-org/GLM-4.5",
    ),
1509
1510
1511
    "Glm4MoeLiteMTPModel": _HfExamplesInfo(
        "zai-org/GLM-4.7-Flash",
        speculative_model="zai-org/GLM-4.7-Flash",
1512
        min_transformers_version="5.0.0",
1513
1514
1515
1516
    ),
    "GlmOcrMTPModel": _HfExamplesInfo(
        "zai-org/GLM-OCR",
        speculative_model="zai-org/GLM-OCR",
1517
        is_available_online=False,
1518
        min_transformers_version="5.1.0",
1519
    ),
1520
1521
1522
1523
    "HYV3MTPModel": _HfExamplesInfo(
        "tencent/Hy3-preview",
        speculative_model="tencent/Hy3-preview",
    ),
XuruiYang's avatar
XuruiYang committed
1524
1525
1526
    "LongCatFlashMTPModel": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat",
        trust_remote_code=True,
1527
1528
1529
1530
1531
1532
1533
        speculative_model="meituan-longcat/LongCat-Flash-Chat",
    ),
    "MiMoMTPModel": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-7B-RL",
        trust_remote_code=True,
        speculative_model="XiaomiMiMo/MiMo-7B-RL",
    ),
1534
1535
1536
1537
    "NemotronHMTPModel": _HfExamplesInfo(
        "nvidia/Nemotron-Super-Placeholder",
        speculative_model="nvidia/Nemotron-Super-Placeholder",
        is_available_online=False,
1538
    ),
1539
1540
1541
1542
    "OpenPanguMTPModel": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
1543
    ),
1544
1545
1546
    "Qwen3NextMTP": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
    ),
1547
1548
1549
1550
1551
1552
1553
1554
    "Qwen3_5MTP": _HfExamplesInfo(
        "Qwen/Qwen3.5-0.8B",
        speculative_model="Qwen/Qwen3.5-0.8B",
    ),
    "Qwen3_5MoeMTP": _HfExamplesInfo(
        "Qwen/Qwen3.5-35B-A3B",
        speculative_model="Qwen/Qwen3.5-35B-A3B",
    ),
csy0225's avatar
csy0225 committed
1555
1556
1557
    "Step3p5MTP": _HfExamplesInfo(
        "stepfun-ai/Step-3.5-Flash",
        speculative_model="stepfun-ai/Step-3.5-Flash",
1558
1559
        use_original_num_layers=True,
        # Initialize at least one MoE layer
1560
        hf_overrides={"num_hidden_layers": 4},
csy0225's avatar
csy0225 committed
1561
1562
        is_available_online=False,
    ),
1563
1564
}

1565
_TRANSFORMERS_BACKEND_MODELS = {
1566
    "TransformersEmbeddingModel": _HfExamplesInfo(
1567
        "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
1568
1569
1570
    ),
    "TransformersForSequenceClassification": _HfExamplesInfo(
        "papluca/xlm-roberta-base-language-detection",
1571
        min_transformers_version="5.0.0",
1572
1573
1574
1575
    ),
    "TransformersForCausalLM": _HfExamplesInfo(
        "hmellor/Ilama-3.2-1B", trust_remote_code=True
    ),
1576
    "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
1577
    "TransformersMoEForCausalLM": _HfExamplesInfo(
1578
        "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
1579
    ),
1580
    "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
1581
        "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
1582
1583
    ),
    "TransformersMoEEmbeddingModel": _HfExamplesInfo(
1584
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
1585
1586
    ),
    "TransformersMoEForSequenceClassification": _HfExamplesInfo(
1587
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
1588
    ),
1589
1590
1591
1592
    "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
    "TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
        "google/gemma-3-4b-it"
    ),
1593
1594
}

1595
1596
1597
_EXAMPLE_MODELS = {
    **_TEXT_GENERATION_EXAMPLE_MODELS,
    **_EMBEDDING_EXAMPLE_MODELS,
1598
1599
1600
    **_LATE_INTERACTION_EXAMPLE_MODELS,
    **_REWARD_EXAMPLE_MODELS,
    **_TOKEN_CLASSIFICATION_EXAMPLE_MODELS,
1601
    **_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS,
1602
1603
    **_MULTIMODAL_EXAMPLE_MODELS,
    **_SPECULATIVE_DECODING_EXAMPLE_MODELS,
1604
    **_TRANSFORMERS_BACKEND_MODELS,
1605
1606
1607
1608
1609
1610
1611
1612
1613
}


class HfExampleModels:
    def __init__(self, hf_models: Mapping[str, _HfExamplesInfo]) -> None:
        super().__init__()

        self.hf_models = hf_models

1614
    def get_supported_archs(self) -> Set[str]:
1615
1616
1617
        return self.hf_models.keys()

    def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
1618
1619
1620
        try:
            return self.hf_models[model_arch]
        except KeyError:
1621
1622
1623
            raise ValueError(
                f"No example model defined for {model_arch}; please update this file."
            ) from None
1624

1625
1626
1627
1628
1629
    def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
        for info in self.hf_models.values():
            if info.default == model_id:
                return info

1630
1631
1632
1633
1634
        # Fallback to extras
        for info in self.hf_models.values():
            if any(extra == model_id for extra in info.extras.values()):
                return info

1635
1636
1637
        raise ValueError(
            f"No example model defined for {model_id}; please update this file."
        )
1638

1639

Patrick von Platen's avatar
Patrick von Platen committed
1640
HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
1641
AUTO_EXAMPLE_MODELS = HfExampleModels(_AUTOMATIC_CONVERTED_MODELS)