registry.py 54.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Mapping, Set
5
from dataclasses import dataclass, field
6
from typing import Any, Literal
7
8
9

import pytest
from packaging.version import Version
10
from transformers import PretrainedConfig
11
from transformers import __version__ as TRANSFORMERS_VERSION
12

13
from vllm.config.model import ModelDType, TokenizerMode
14

15
16
17
18
19
20
21
22
23

@dataclass(frozen=True)
class _HfExamplesInfo:
    default: str
    """The default model to use for testing this architecture."""

    extras: Mapping[str, str] = field(default_factory=dict)
    """Extra models to use for testing this architecture."""

24
    tokenizer: str | None = None
25
26
    """Set the tokenizer to load for this architecture."""

27
    tokenizer_mode: TokenizerMode | str = "auto"
28
29
    """Set the tokenizer type for this architecture."""

30
    speculative_model: str | None = None
31
32
33
34
35
    """
    The default model to use for testing this architecture, which is only used
    for speculative decoding.
    """

36
37
38
39
40
    speculative_method: str | None = None
    """
    The method to use for speculative decoding.
    """

41
    min_transformers_version: str | None = None
42
43
44
45
    """
    The minimum version of HF Transformers that is required to run this model.
    """

46
    max_transformers_version: str | None = None
47
48
49
50
    """
    The maximum version of HF Transformers that this model runs on.
    """

51
    transformers_version_reason: dict[Literal["vllm", "hf"], str] | None = None
52
    """
53
54
55
    The type and reason to skip test for the minimum/maximum version requirement.
    vllm: skip all vLLM tests if the version requirement is not met.
    hf: only skip tests that uses HF runner if the version requirement is not met.
56
57
    """

58
    require_embed_inputs: bool = False
59
    """
60
61
    If `True`, enables prompt and multi-modal embedding inputs while
    disabling tokenization.
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    """

    dtype: ModelDType = "auto"
    """
    The data type for the model weights and activations.
    """

    enforce_eager: bool = False
    """
    Whether to enforce eager execution. If True, we will
    disable CUDA graph and always execute the model in eager mode.
    If False, we will use CUDA graph and eager execution in hybrid.
    """

76
77
78
79
80
81
    enable_prefix_caching: bool = True
    """
    Whether to enable prefix caching for the model. If True, we will test the model with
    prefix caching enabled. If False, we will test the model without prefix caching.
    """

82
83
    is_available_online: bool = True
    """
84
    Set this to `False` if the name of this architecture no longer exists on
85
86
87
88
89
90
    the HF repo. To maintain backwards compatibility, we have not removed them
    from the main model registry, so without this flag the registry tests will
    fail.
    """

    trust_remote_code: bool = False
91
    """The `trust_remote_code` level required to load the model."""
92

93
    hf_overrides: dict[str, Any] = field(default_factory=dict)
94
    """The `hf_overrides` required to load the model."""
95

96
    max_model_len: int | None = None
97
98
99
100
101
    """
    The maximum model length to use for this model. Some models default to a
    length that is too large to fit into memory in CI.
    """

102
103
104
105
106
    max_num_batched_tokens: int | None = None
    """
    The maximum number of tokens to be processed in a single batch.
    """

107
    revision: str | None = None
108
109
110
111
112
    """
    The specific revision (commit hash, tag, or branch) to use for the model.
    If not specified, the default revision will be used.
    """

113
    max_num_seqs: int | None = None
114
115
    """Maximum number of sequences to be processed in a single iteration."""

116
117
    use_original_num_layers: bool = False
    """
118
    If True, use the original number of layers from the model config
119
120
121
    instead of minimal layers for testing.
    """

122
123
124
    def check_transformers_version(
        self,
        *,
125
        on_fail: Literal["error", "skip", "return"],
126
        check_version_reason: Literal["vllm", "hf"] = "hf",
127
128
        check_min_version: bool = True,
        check_max_version: bool = True,
129
    ) -> str | None:
130
131
132
133
        """
        If the installed transformers version does not meet the requirements,
        perform the given action.
        """
134
135
136
137
        if (
            self.min_transformers_version is None
            and self.max_transformers_version is None
        ):
138
            return None
139
140

        current_version = TRANSFORMERS_VERSION
141
        cur_base_version = Version(current_version).base_version
142
143
144
        min_version = self.min_transformers_version
        max_version = self.max_transformers_version
        msg = f"`transformers=={current_version}` installed, but `transformers"
145
146
        # Only check the base version for the min/max version, otherwise preview
        # models cannot be run because `x.yy.0.dev0`<`x.yy.0`
147
148
        if min_version and Version(cur_base_version) < Version(min_version):
            is_version_valid = not check_min_version
149
            msg += f">={min_version}` is required to run this model."
150
151
        elif max_version and Version(cur_base_version) > Version(max_version):
            is_version_valid = not check_max_version
152
153
            msg += f"<={max_version}` is required to run this model."
        else:
154
155
156
157
158
159
160
161
162
163
164
            is_version_valid = True

        # check if Transformers version breaks the corresponding model runner,
        # skip test when model runner not compatible
        is_reason_valid = not (
            check_version_reason
            and self.transformers_version_reason
            and check_version_reason in self.transformers_version_reason
        )
        is_transformers_valid = is_version_valid and is_reason_valid
        if is_transformers_valid:
165
            return None
166
167
168
        elif self.transformers_version_reason:
            for reason_type, reason in self.transformers_version_reason.items():
                msg += f" Reason({reason_type}): {reason}"
169
170
171

        if on_fail == "error":
            raise RuntimeError(msg)
172
        elif on_fail == "skip":
173
            pytest.skip(msg)
174

175
176
        return msg

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
    def check_available_online(
        self,
        *,
        on_fail: Literal["error", "skip"],
    ) -> None:
        """
        If the model is not available online, perform the given action.
        """
        if not self.is_available_online:
            msg = "Model is not available online"

            if on_fail == "error":
                raise RuntimeError(msg)
            else:
                pytest.skip(msg)

193
194
195

_TEXT_GENERATION_EXAMPLE_MODELS = {
    # [Decoder-only]
196
    "AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
197
    "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
198
199
    "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B", trust_remote_code=True),
    "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B", trust_remote_code=True),
200
    "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
201
202
203
    "ArcticForCausalLM": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-instruct", trust_remote_code=True
    ),
204
    "AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
205
206
207
208
209
210
211
212
213
214
215
216
    "BaiChuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan-7B", trust_remote_code=True
    ),
    "BaichuanForCausalLM": _HfExamplesInfo(
        "baichuan-inc/Baichuan2-7B-chat", trust_remote_code=True
    ),
    "BailingMoeForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-lite-1.5", trust_remote_code=True
    ),
    "BailingMoeV2ForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ling-mini-2.0", trust_remote_code=True
    ),
Jiangyun Zhu's avatar
Jiangyun Zhu committed
217
218
219
    "BailingMoeV2_5ForCausalLM": _HfExamplesInfo(
        "inclusionAI/Ring-2.5-1T", trust_remote_code=True
    ),
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
    "BambaForCausalLM": _HfExamplesInfo(
        "ibm-ai-platform/Bamba-9B-v1",
        extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"},
    ),
    "BloomForCausalLM": _HfExamplesInfo(
        "bigscience/bloom-560m", {"1b": "bigscience/bloomz-1b1"}
    ),
    "ChatGLMModel": _HfExamplesInfo(
        "zai-org/chatglm3-6b", trust_remote_code=True, max_transformers_version="4.48"
    ),
    "ChatGLMForConditionalGeneration": _HfExamplesInfo(
        "thu-coai/ShieldLM-6B-chatglm3",
        trust_remote_code=True,
    ),
    "CohereForCausalLM": _HfExamplesInfo(
235
        "CohereLabs/c4ai-command-r-v01", trust_remote_code=True
236
237
    ),
    "Cohere2ForCausalLM": _HfExamplesInfo(
238
        "CohereLabs/c4ai-command-r7b-12-2024",
239
240
        trust_remote_code=True,
    ),
241
    "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
242
243
244
245
    # FIXME: databricks/dbrx-instruct has been deleted
    "DbrxForCausalLM": _HfExamplesInfo(
        "databricks/dbrx-instruct", is_available_online=False
    ),
246
247
248
249
    "DeciLMForCausalLM": _HfExamplesInfo(
        "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
        trust_remote_code=True,
    ),
250
251
252
253
    "DeepseekForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-moe-16b-base",
        trust_remote_code=True,
    ),
254
255
256
257
258
259
260
261
    "DeepseekV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V2-Lite-Chat",
        trust_remote_code=True,
    ),
    "DeepseekV3ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-V3",
        trust_remote_code=True,
    ),
262
    "DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
263
264
    "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"),
    "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"),
265
266
267
    "ExaoneForCausalLM": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
    ),
268
    "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
Kyungmin Lee's avatar
Kyungmin Lee committed
269
    "ExaoneMoEForCausalLM": _HfExamplesInfo(
270
        "LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
Kyungmin Lee's avatar
Kyungmin Lee committed
271
    ),
272
    "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
273
    "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
274
    "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
275
    "FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
276
    "GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
277
278
279
    "Gemma2ForCausalLM": _HfExamplesInfo(
        "google/gemma-2-9b", extras={"tiny": "google/gemma-2-2b-it"}
    ),
280
    "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
281
282
283
284
    "Gemma4ForCausalLM": _HfExamplesInfo(
        "google/gemma-4-E2B-it",
        min_transformers_version="5.0.0",
    ),
285
    "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
286
287
    "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
    "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
288
    "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
289
290
    "Glm4MoeLiteForCausalLM": _HfExamplesInfo(
        "zai-org/GLM-4.7-Flash",
291
        min_transformers_version="5.0.0",
292
    ),
Jee Jee Li's avatar
Jee Jee Li committed
293
294
295
    "GlmMoeDsaForCausalLM": _HfExamplesInfo(
        "zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
    ),
296
297
298
    "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
    "GPTBigCodeForCausalLM": _HfExamplesInfo(
        "bigcode/starcoder",
299
300
301
302
        extras={
            "tiny": "bigcode/tiny_starcoder_py",
            "santacoder": "bigcode/gpt_bigcode-santacoder",
        },
303
304
305
306
307
308
309
    ),
    "GPTJForCausalLM": _HfExamplesInfo(
        "Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"}
    ),
    "GPTNeoXForCausalLM": _HfExamplesInfo(
        "EleutherAI/pythia-70m", {"1b": "EleutherAI/pythia-1.4b"}
    ),
310
    "GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"),
311
312
    "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
    "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
313
    "GraniteMoeHybridForCausalLM": _HfExamplesInfo(
314
        "ibm-granite/granite-4.0-tiny-preview"
315
316
317
318
319
320
321
    ),
    "GraniteMoeSharedForCausalLM": _HfExamplesInfo(
        "ibm-research/moe-7b-1b-active-shared-experts"
    ),
    "Grok1ModelForCausalLM": _HfExamplesInfo(
        "hpcai-tech/grok-1", trust_remote_code=True
    ),
Bijaya Dangol's avatar
Bijaya Dangol committed
322
    "Grok1ForCausalLM": _HfExamplesInfo("xai-org/grok-2", trust_remote_code=True),
323
    "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
324
325
326
    "HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
        "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
    ),
327
    "HyperCLOVAXForCausalLM": _HfExamplesInfo(
328
        "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
329
330
        trust_remote_code=True,
    ),
331
332
333
334
335
336
337
338
339
340
341
342
    "InternLMForCausalLM": _HfExamplesInfo(
        "internlm/internlm-chat-7b", trust_remote_code=True
    ),
    "InternLM2ForCausalLM": _HfExamplesInfo(
        "internlm/internlm2-chat-7b", trust_remote_code=True
    ),
    "InternLM2VEForCausalLM": _HfExamplesInfo(
        "OpenGVLab/Mono-InternVL-2B", trust_remote_code=True
    ),
    "InternLM3ForCausalLM": _HfExamplesInfo(
        "internlm/internlm3-8b-instruct", trust_remote_code=True
    ),
343
344
345
346
347
348
    "IQuestCoderForCausalLM": _HfExamplesInfo(
        "IQuestLab/IQuest-Coder-V1-40B-Instruct", trust_remote_code=True
    ),
    "IQuestLoopCoderForCausalLM": _HfExamplesInfo(
        "IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
    ),
349
    "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
350
351
352
    "Jais2ForCausalLM": _HfExamplesInfo(
        "inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
    ),
353
354
355
356
357
358
359
    "JambaForCausalLM": _HfExamplesInfo(
        "ai21labs/AI21-Jamba-1.5-Mini",
        extras={
            "tiny": "ai21labs/Jamba-tiny-dev",
            "random": "ai21labs/Jamba-tiny-random",
        },
    ),
360
361
362
    "KimiLinearForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True
    ),
363
    "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
Paul Pak's avatar
Paul Pak committed
364
    "Lfm2MoeForCausalLM": _HfExamplesInfo(
365
366
367
368
369
        "LiquidAI/LFM2-8B-A1B",
        min_transformers_version="5.0.0",
        use_original_num_layers=True,
        # Initialize at least one MoE layer
        hf_overrides={"num_hidden_layers": 4},
Paul Pak's avatar
Paul Pak committed
370
    ),
371
372
373
374
375
376
    "LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.2-1B-Instruct",
        extras={
            "guard": "meta-llama/Llama-Guard-3-1B",
            "hermes": "NousResearch/Hermes-3-Llama-3.1-8B",
            "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
377
            "tiny": "hmellor/tiny-random-LlamaForCausalLM",
378
379
380
381
382
383
384
385
386
387
388
        },
    ),
    "LLaMAForCausalLM": _HfExamplesInfo(
        "decapoda-research/llama-7b-hf", is_available_online=False
    ),
    "Llama4ForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "LongcatFlashForCausalLM": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
    ),
389
    "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
390
391
392
393
394
395
396
397
398
399
400
401
402
    "Mamba2ForCausalLM": _HfExamplesInfo(
        "mistralai/Mamba-Codestral-7B-v0.1",
        extras={
            "random": "yujiepan/mamba2-codestral-v0.1-tiny-random",
        },
    ),
    "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"),
    "MiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-2B-sft-bf16", trust_remote_code=True
    ),
    "MiniCPM3ForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM3-4B", trust_remote_code=True
    ),
403
404
405
    "MiniCPM4ForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM4.1-8B", trust_remote_code=True
    ),
406
    "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
407
408
409
410
411
412
413
414
    "MiniMaxText01ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-Text-01",
        trust_remote_code=True,
        revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3",
    ),
    "MiniMaxM1ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-M1-40k", trust_remote_code=True
    ),
415
    "MiniMaxM2ForCausalLM": _HfExamplesInfo(
youkaichao's avatar
youkaichao committed
416
417
        "MiniMaxAI/MiniMax-M2",
        trust_remote_code=True,
418
    ),
419
    "Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"),
420
    "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
421
    "MistralLarge3ForCausalLM": _HfExamplesInfo(
422
        "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4"
423
    ),
424
425
426
427
    "MixtralForCausalLM": _HfExamplesInfo(
        "mistralai/Mixtral-8x7B-Instruct-v0.1",
        {"tiny": "TitanML/tiny-mixtral"},
    ),
428
    "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
429
430
    # FIXME: mosaicml/mpt-7b has been deleted
    "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b", is_available_online=False),
431
    "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
432
433
434
    "NemotronHForCausalLM": _HfExamplesInfo(
        "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
    ),
435
436
437
438
439
    "NemotronHPuzzleForCausalLM": _HfExamplesInfo(
        "",
        trust_remote_code=True,
        is_available_online=False,
    ),
440
    "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
441
    "Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
442
    "Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
443
    "OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"),
444
    "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
445
446
447
448
449
450
    "OPTForCausalLM": _HfExamplesInfo(
        "facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
    ),
    "OrionForCausalLM": _HfExamplesInfo(
        "OrionStarAI/Orion-14B-Chat", trust_remote_code=True
    ),
451
    "OuroForCausalLM": _HfExamplesInfo("ByteDance/Ouro-1.4B", trust_remote_code=True),
452
453
454
    "PanguEmbeddedForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Embedded-7B-V1.1", trust_remote_code=True
    ),
455
456
457
458
459
    "PanguProMoEV2ForCausalLM": _HfExamplesInfo(
        "",
        trust_remote_code=True,
        is_available_online=False,
    ),
460
461
462
463
464
    "PanguUltraMoEForCausalLM": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
    ),
465
466
467
468
    "Param2MoEForCausalLM": _HfExamplesInfo(
        "bharatgenai/Param2-17B-A2.4B-Thinking",
        trust_remote_code=True,
    ),
469
    "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
470
    "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
471
    "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
472
473
474
475
476
477
478
    "PhiMoEForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3.5-MoE-instruct", trust_remote_code=True
    ),
    "Plamo2ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-2-1b",
        trust_remote_code=True,
    ),
479
480
481
482
    "Plamo3ForCausalLM": _HfExamplesInfo(
        "pfnet/plamo-3-nict-2b-base",
        trust_remote_code=True,
    ),
483
484
485
    "QWenLMHeadModel": _HfExamplesInfo(
        "Qwen/Qwen-7B-Chat",
        max_transformers_version="4.53",
486
487
488
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
489
490
491
        trust_remote_code=True,
    ),
    "Qwen2ForCausalLM": _HfExamplesInfo(
492
493
494
495
496
        "Qwen/Qwen2-0.5B-Instruct",
        extras={
            "2.5": "Qwen/Qwen2.5-0.5B-Instruct",
            "2.5-1.5B": "Qwen/Qwen2.5-1.5B-Instruct",
        },
497
    ),
498
    "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
Jee Jee Li's avatar
Jee Jee Li committed
499
500
    "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
    "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
501
502
503
504
505
    "Qwen3NextForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct",
        extras={"tiny-random": "tiny-random/qwen3-next-moe"},
        min_transformers_version="4.56.3",
    ),
506
    "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
507
508
509
510
511
512
513
514
515
516
517
518
    "SarvamMoEForCausalLM": _HfExamplesInfo(
        "sarvamai/sarvam-30b",
        trust_remote_code=True,
        max_model_len=4096,
        is_available_online=True,
    ),
    "SarvamMLAForCausalLM": _HfExamplesInfo(
        "sarvamai/sarvam-105b",
        trust_remote_code=True,
        max_model_len=4096,
        is_available_online=True,
    ),
519
520
521
522
    "SeedOssForCausalLM": _HfExamplesInfo(
        "ByteDance-Seed/Seed-OSS-36B-Instruct",
        trust_remote_code=True,
    ),
523
524
525
526
    "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
    "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
    "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
    "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
Li Xie's avatar
Li Xie committed
527
528
529
    "Step1ForCausalLM": _HfExamplesInfo(
        "stepfun-ai/Step-Audio-EditX", trust_remote_code=True
    ),
csy0225's avatar
csy0225 committed
530
    "Step3p5ForCausalLM": _HfExamplesInfo(
531
532
533
        "stepfun-ai/Step-3.5-Flash",
        use_original_num_layers=True,
        # Initialize at least one MoE layer
534
        hf_overrides={"num_hidden_layers": 4},
csy0225's avatar
csy0225 committed
535
    ),
536
537
538
539
    "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3", trust_remote_code=True),
    "SolarForCausalLM": _HfExamplesInfo(
        "upstage/solar-pro-preview-instruct", trust_remote_code=True
    ),
540
541
542
    "TeleChatForCausalLM": _HfExamplesInfo(
        "chuhac/TeleChat2-35B", trust_remote_code=True
    ),
543
544
545
    "TeleChat2ForCausalLM": _HfExamplesInfo(
        "Tele-AI/TeleChat2-3B", trust_remote_code=True
    ),
546
547
548
    "TeleChat3ForCausalLM": _HfExamplesInfo(
        "Tele-AI/TeleChat3-36B-Thinking", trust_remote_code=True
    ),
549
550
551
552
553
554
555
556
    "TeleFLMForCausalLM": _HfExamplesInfo(
        "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
    ),
    "XverseForCausalLM": _HfExamplesInfo(
        "xverse/XVERSE-7B-Chat",
        tokenizer="meta-llama/Llama-2-7b",
        trust_remote_code=True,
    ),
557
    "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
558
    "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
559
560
561
    "MiMoV2FlashForCausalLM": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-V2-Flash", trust_remote_code=True
    ),
562
    "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
563
564
565
566
}

_EMBEDDING_EXAMPLE_MODELS = {
    # [Text-only]
567
    "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
568
    "ErnieModel": _HfExamplesInfo("shibing624/text2vec-base-chinese-sentence"),
569
570
571
    "BertSpladeSparseEmbeddingModel": _HfExamplesInfo(
        "naver/splade-v3",
        hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
572
    ),
573
    "BgeM3EmbeddingModel": _HfExamplesInfo("BAAI/bge-m3"),
574
    "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
575
    "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
576
    "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
577
578
579
580
581
582
583
584
    "GteModel": _HfExamplesInfo(
        "Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True
    ),
    "GteNewModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-base-en-v1.5",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewModel"]},
    ),
585
    "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
586
587
588
    "LlamaBidirectionalModel": _HfExamplesInfo(
        "nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True
    ),
589
    "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
590
591
592
593
594
595
    "ModernBertModel": _HfExamplesInfo(
        "Alibaba-NLP/gte-modernbert-base", trust_remote_code=True
    ),
    "NomicBertModel": _HfExamplesInfo(
        "nomic-ai/nomic-embed-text-v2-moe", trust_remote_code=True
    ),
596
    "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
597
598
    "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"),
    "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"),
chengchengpei's avatar
chengchengpei committed
599
600
601
    "VoyageQwen3BidirectionalEmbedModel": _HfExamplesInfo(
        "voyageai/voyage-4-nano", trust_remote_code=True
    ),
602
    "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"),
603
    # [Multimodal]
604
    "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
605
606
607
    "LlamaNemotronVLModel": _HfExamplesInfo(
        "nvidia/llama-nemotron-embed-vl-1b-v2", trust_remote_code=True
    ),
608
    "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
609
610
611
612
    "Phi3VForCausalLM": _HfExamplesInfo(
        "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
613
    "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"),
614
615
    "PrithviGeoSpatialMAE": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
616
        dtype="float16",
617
        enforce_eager=True,
618
619
        require_embed_inputs=True,
        # This is to avoid the model going OOM in CI
620
621
622
623
        max_num_seqs=32,
    ),
    "Terratorch": _HfExamplesInfo(
        "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
624
        dtype="float16",
625
        enforce_eager=True,
626
        require_embed_inputs=True,
627
628
629
        # This is to avoid the model going OOM in CI
        max_num_seqs=32,
    ),
630
631
}

632
633
634
635
636
637
_LATE_INTERACTION_EXAMPLE_MODELS = {
    # [Text-only]
    "HF_ColBERT": _HfExamplesInfo("answerdotai/answerai-colbert-small-v1"),
    "ColBERTModernBertModel": _HfExamplesInfo(
        "lightonai/GTE-ModernColBERT-v1",
        hf_overrides={"architectures": ["ColBERTModernBertModel"]},
638
    ),
639
640
641
642
643
    "ColBERTJinaRobertaModel": _HfExamplesInfo(
        "jinaai/jina-colbert-v2",
        trust_remote_code=True,
        hf_overrides={"architectures": ["ColBERTJinaRobertaModel"]},
    ),
644
645
646
647
648
    "ColBERTLfm2Model": _HfExamplesInfo(
        "LiquidAI/LFM2-ColBERT-350M",
        trust_remote_code=True,
        hf_overrides={"architectures": ["ColBERTLfm2Model"]},
    ),
649
    "JinaForRanking": _HfExamplesInfo("jinaai/jina-reranker-v3"),
650
651
652
653
    # [Multimodal]
    "ColModernVBertForRetrieval": _HfExamplesInfo(
        "ModernVBERT/colmodernvbert-merged",
    ),
654
    "ColPaliForRetrieval": _HfExamplesInfo("vidore/colpali-v1.3-hf"),
655
656
657
658
659
660
    "ColQwen3": _HfExamplesInfo(
        "TomoroAI/tomoro-colqwen3-embed-4b", trust_remote_code=True
    ),
    "OpsColQwen3Model": _HfExamplesInfo(
        "OpenSearch-AI/Ops-Colqwen3-4B", trust_remote_code=True
    ),
661
662
663
664
665
    "ColQwen3_5": _HfExamplesInfo(
        "athrael-soju/colqwen3.5-4.5B-v3",
        trust_remote_code=True,
        max_model_len=4096,
    ),
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
    "Qwen3VLNemotronEmbedModel": _HfExamplesInfo(
        "nvidia/nemotron-colembed-vl-4b-v2",
    ),
}


_REWARD_EXAMPLE_MODELS = {
    "InternLM2ForRewardModel": _HfExamplesInfo(
        "internlm/internlm2-1_8b-reward", trust_remote_code=True
    ),
    "Qwen2ForRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-RM-72B",
        max_transformers_version="4.53",
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
    ),
    "Qwen2ForProcessRewardModel": _HfExamplesInfo(
        "Qwen/Qwen2.5-Math-PRM-7B",
        max_transformers_version="4.53",
        transformers_version_reason={
            "hf": "HF model uses remote code that is not compatible with latest Transformers"  # noqa: E501
        },
    ),
}

_TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
    "BertForTokenClassification": _HfExamplesInfo("boltuix/NeuroBERT-NER"),
694
695
696
    "ErnieForTokenClassification": _HfExamplesInfo(
        "gyr66/Ernie-3.0-base-chinese-finetuned-ner"
    ),
697
698
699
700
701
702
    "ModernBertForTokenClassification": _HfExamplesInfo(
        "disham993/electrical-ner-ModernBERT-base"
    ),
}

_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
703
704
705
    "BertForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/ms-marco-MiniLM-L-6-v2"
    ),
706
707
708
    "ErnieForSequenceClassification": _HfExamplesInfo(
        "Forrest20231206/ernie-3.0-base-zh-cls",
    ),
709
710
711
    "GPT2ForSequenceClassification": _HfExamplesInfo(
        "nie3e/sentiment-polish-gpt2-small"
    ),
712
713
714
715
716
    "GteNewForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-multilingual-reranker-base",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
    ),
717
    "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"),
718
719
720
    "LlamaBidirectionalForSequenceClassification": _HfExamplesInfo(
        "nvidia/llama-nemotron-rerank-1b-v2", trust_remote_code=True
    ),
721
722
723
    "LlamaNemotronVLForSequenceClassification": _HfExamplesInfo(
        "nvidia/llama-nemotron-rerank-vl-1b-v2", trust_remote_code=True
    ),
724
725
726
727
728
729
730
    "ModernBertForSequenceClassification": _HfExamplesInfo(
        "Alibaba-NLP/gte-reranker-modernbert-base"
    ),
    "RobertaForSequenceClassification": _HfExamplesInfo(
        "cross-encoder/quora-roberta-base"
    ),
    "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"),
731
732
}

733
734
_AUTOMATIC_CONVERTED_MODELS = {
    # Use as_seq_cls_model for automatic conversion
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
    "GemmaForSequenceClassification": _HfExamplesInfo(
        "BAAI/bge-reranker-v2-gemma",
        hf_overrides={
            "architectures": ["GemmaForSequenceClassification"],
            "classifier_from_token": ["Yes"],
            "method": "no_post_processing",
        },
    ),
    "LlamaForSequenceClassification": _HfExamplesInfo(
        "Skywork/Skywork-Reward-V2-Llama-3.2-1B"
    ),
    "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"),
    "Qwen3ForSequenceClassification": _HfExamplesInfo(
        "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
    ),
750
    "Qwen3ForTokenClassification": _HfExamplesInfo("bd2lcco/Qwen3-0.6B-finetuned"),
751
752
753
754
755
756
757
758
759
    "Qwen3VLForSequenceClassification": _HfExamplesInfo(
        "Qwen/Qwen3-VL-Reranker-2B",
        is_available_online=False,
        hf_overrides={
            "architectures": ["Qwen3VLForSequenceClassification"],
            "classifier_from_token": ["no", "yes"],
            "is_original_qwen3_reranker": True,
        },
    ),
760
761
}

762
763
_MULTIMODAL_EXAMPLE_MODELS = {
    # [Decoder-only]
764
    "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
765
    "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
766
        "nvidia/audio-flamingo-3-hf", min_transformers_version="5.0.0"
767
    ),
768
    "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
769
        "nvidia/music-flamingo-2601-hf", min_transformers_version="5.3.0"
770
    ),
771
    "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
772
    "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
773
774
775
776
    "BeeForConditionalGeneration": _HfExamplesInfo(
        "Open-Bee/Bee-8B-RL",
        trust_remote_code=True,
    ),
777
778
779
780
781
    "Blip2ForConditionalGeneration": _HfExamplesInfo(
        "Salesforce/blip2-opt-2.7b",
        extras={"6b": "Salesforce/blip2-opt-6.7b"},
    ),
    "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),
782
783
784
785
786
787
788
789
    "Cheers": _HfExamplesInfo(
        "ai9stars/Cheers",
        trust_remote_code=True,
    ),
    "CheersForConditionalGeneration": _HfExamplesInfo(
        "ai9stars/Cheers",
        trust_remote_code=True,
    ),
790
791
792
793
794
795
796
    "Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
        "CohereLabs/command-a-vision-07-2025"
    ),
    "DeepseekVLV2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/deepseek-vl2-tiny",
        extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
        max_transformers_version="4.48",
797
        transformers_version_reason={"hf": "HF model is not compatible."},
798
    ),
799
800
801
    "DeepseekOCRForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-OCR",
    ),
RED's avatar
RED committed
802
803
804
    "DeepseekOCR2ForCausalLM": _HfExamplesInfo(
        "deepseek-ai/DeepSeek-OCR-2",
    ),
805
806
807
    "DotsOCRForCausalLM": _HfExamplesInfo(
        "rednote-hilab/dots.ocr", trust_remote_code=True
    ),
808
    "Eagle2_5_VLForConditionalGeneration": _HfExamplesInfo(
809
810
        "nvidia/Eagle2.5-8B",
        trust_remote_code=True,
811
    ),
812
    "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
813
814
815
    "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
        "baidu/ERNIE-4.5-VL-28B-A3B-PT",
        trust_remote_code=True,
816
        revision="refs/pr/17",
817
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
818
819
820
821
    "Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-4.5-33B",
        min_transformers_version="5.6.0",
    ),
822
823
824
    "FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
        "allendou/FireRedASR2-LLM-vllm",
    ),
825
826
827
    "FireRedLIDForConditionalGeneration": _HfExamplesInfo(
        "PatchyTisa/FireRedLID-vllm",
    ),
828
829
830
    "FunASRForConditionalGeneration": _HfExamplesInfo(
        "allendou/Fun-ASR-Nano-2512-vllm",
    ),
831
832
833
    "FunAudioChatForConditionalGeneration": _HfExamplesInfo(
        "funaudiochat", is_available_online=False
    ),
834
    "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
835
    "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
836
837
838
839
    "Gemma4ForConditionalGeneration": _HfExamplesInfo(
        "google/gemma-4-E2B-it",
        min_transformers_version="5.5.0",
    ),
840
    "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
841
842
    "GlmAsrForConditionalGeneration": _HfExamplesInfo(
        "zai-org/GLM-ASR-Nano-2512",
843
        min_transformers_version="5.0.0",
844
    ),
845
    "GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"),
846
    "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
847
848
        "ibm-granite/granite-speech-3.3-2b",
        extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"},
849
850
851
852
853
854
855
    ),
    "GLM4VForCausalLM": _HfExamplesInfo(
        "zai-org/glm-4v-9b",
        trust_remote_code=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
    ),
    "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
856
    "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
857
858
    "GlmOcrForConditionalGeneration": _HfExamplesInfo(
        "zai-org/GLM-OCR",
859
        min_transformers_version="5.1.0",
860
    ),
861
862
863
864
865
    "H2OVLChatModel": _HfExamplesInfo(
        "h2oai/h2ovl-mississippi-800m",
        trust_remote_code=True,
        extras={"2b": "h2oai/h2ovl-mississippi-2b"},
        max_transformers_version="4.48",
866
        transformers_version_reason={"hf": "HF model is not compatible."},
867
868
869
870
871
    ),
    "HCXVisionForCausalLM": _HfExamplesInfo(
        "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
        trust_remote_code=True,
    ),
872
873
874
875
    "HCXVisionV2ForCausalLM": _HfExamplesInfo(
        "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
        trust_remote_code=True,
    ),
876
877
    "HunYuanVLForConditionalGeneration": _HfExamplesInfo(
        "tencent/HunyuanOCR",
878
        hf_overrides={"num_experts": 0},
879
    ),
880
881
    "Idefics3ForConditionalGeneration": _HfExamplesInfo(
        "HuggingFaceM4/Idefics3-8B-Llama3",
882
        extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
883
    ),
oscardev256's avatar
oscardev256 committed
884
885
886
    "IsaacForConditionalGeneration": _HfExamplesInfo(
        "PerceptronAI/Isaac-0.1",
        trust_remote_code=True,
887
        extras={"0.2-2B-Preview": "PerceptronAI/Isaac-0.2-2B-Preview"},
oscardev256's avatar
oscardev256 committed
888
    ),
889
890
891
    "InternS1ForConditionalGeneration": _HfExamplesInfo(
        "internlm/Intern-S1", trust_remote_code=True
    ),
zxy's avatar
zxy committed
892
893
894
895
    "InternS1ProForConditionalGeneration": _HfExamplesInfo(
        "internlm/Intern-S1-Pro",
        trust_remote_code=True,
    ),
896
897
898
899
900
901
902
903
904
905
906
907
    "InternVLChatModel": _HfExamplesInfo(
        "OpenGVLab/InternVL2-1B",
        extras={
            "2B": "OpenGVLab/InternVL2-2B",
            "3.0": "OpenGVLab/InternVL3-1B",
            "3.5-qwen3": "OpenGVLab/InternVL3_5-1B",
            "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B",
            "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
        },
        trust_remote_code=True,
    ),
    "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"),
908
909
910
911
    "KananaVForConditionalGeneration": _HfExamplesInfo(
        "kakaocorp/kanana-1.5-v-3b-instruct",
        trust_remote_code=True,
    ),
912
913
914
915
916
917
918
919
    "KeyeForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-8B-Preview",
        trust_remote_code=True,
    ),
    "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo(
        "Kwai-Keye/Keye-VL-1_5-8B",
        trust_remote_code=True,
    ),
920
921
922
923
924
925
926
927
928
    "MoonshotKimiaForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-Audio-7B-Instruct",
        tokenizer_mode="kimi_audio",
        trust_remote_code=True,
    ),
    "KimiK25ForConditionalGeneration": _HfExamplesInfo(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True,
    ),
929
930
931
932
    "KimiVLForConditionalGeneration": _HfExamplesInfo(
        "moonshotai/Kimi-VL-A3B-Instruct",
        extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
        trust_remote_code=True,
933
        max_transformers_version="4.53.3",
934
935
936
937
938
939
940
        transformers_version_reason={
            "hf": (
                "HF model uses deprecated transformers API "
                "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: "
                "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31"
            )
        },
941
    ),
942
    "LightOnOCRForConditionalGeneration": _HfExamplesInfo(
943
        "lightonai/LightOnOCR-1B-1025"
944
    ),
945
946
947
948
    "Lfm2VlForConditionalGeneration": _HfExamplesInfo(
        "LiquidAI/LFM2-VL-450M",
        min_transformers_version="5.0.0",
    ),
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
    "Llama4ForConditionalGeneration": _HfExamplesInfo(
        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
        max_model_len=10240,
        extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"},
    ),
    "LlavaForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-1.5-7b-hf",
        extras={
            "mistral": "mistral-community/pixtral-12b",
            "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic",
        },
    ),
    "LlavaNextForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-v1.6-mistral-7b-hf"
    ),
    "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/LLaVA-NeXT-Video-7B-hf"
    ),
    "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
        "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
    ),
    "MantisForConditionalGeneration": _HfExamplesInfo(
        "TIGER-Lab/Mantis-8B-siglip-llama3",
        max_transformers_version="4.48",
973
        transformers_version_reason={"hf": "HF model is not compatible."},
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
    ),
    "MiDashengLMModel": _HfExamplesInfo(
        "mispeech/midashenglm-7b", trust_remote_code=True
    ),
    "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6", trust_remote_code=True),
    "MiniCPMV": _HfExamplesInfo(
        "openbmb/MiniCPM-Llama3-V-2_5",
        extras={
            "2.6": "openbmb/MiniCPM-V-2_6",
            "4.0": "openbmb/MiniCPM-V-4",
            "4.5": "openbmb/MiniCPM-V-4_5",
        },
        trust_remote_code=True,
    ),
    "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-VL-01",
        trust_remote_code=True,
    ),
    "Mistral3ForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
        extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
    ),
    "MolmoForCausalLM": _HfExamplesInfo(
        "allenai/Molmo-7B-D-0924",
        max_transformers_version="4.48",
1000
1001
1002
        transformers_version_reason={
            "vllm": "Incorrectly-detected `tensorflow` import from processor."
        },
1003
1004
1005
        extras={"olmo": "allenai/Molmo-7B-O-0924"},
        trust_remote_code=True,
    ),
1006
1007
1008
1009
1010
1011
1012
1013
    "Molmo2ForConditionalGeneration": _HfExamplesInfo(
        "allenai/Molmo2-8B",
        extras={"olmo": "allenai/Molmo2-O-7B"},
        min_transformers_version="4.51",
        trust_remote_code=True,
        # required by current PrefixLM implementation
        max_num_batched_tokens=31872,
    ),
1014
1015
1016
1017
1018
1019
    "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
    "Llama_Nemotron_Nano_VL": _HfExamplesInfo(
        "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
        trust_remote_code=True,
    ),
    "NemotronH_Nano_VL_V2": _HfExamplesInfo(
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
        "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
        max_model_len=4096,
        # NemotronH layers are constructed via `hybrid_override_pattern`:
        use_original_num_layers=True,
        hf_overrides={
            "vision_config": PretrainedConfig(
                args={
                    "min_num_patches": 1,  # Trigger image dynamic res
                    "max_num_patches": 12,
                    "model": "vit_huge_patch16_224",
                },
                # Trigger conv3d:
                video_temporal_patch_size=2,
            ),
            "text_config": {
                "num_hidden_layers": 2,
                "hybrid_override_pattern": "M*",
            },
        },
        trust_remote_code=True,
1040
    ),
Zero's avatar
Zero committed
1041
1042
1043
    "OpenCUAForConditionalGeneration": _HfExamplesInfo(
        "xlangai/OpenCUA-7B", trust_remote_code=True
    ),
1044
1045
1046
1047
1048
1049
    "OpenPanguVLForConditionalGeneration": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-VL-7B",
        trust_remote_code=True,
        max_model_len=4096,
        enforce_eager=True,
    ),
1050
1051
1052
1053
    "Ovis": _HfExamplesInfo(
        "AIDC-AI/Ovis2-1B",
        trust_remote_code=True,
        max_transformers_version="4.53",
1054
        transformers_version_reason={"hf": "HF model is not compatible"},
1055
1056
1057
1058
1059
1060
        extras={
            "1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
            "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
        },
    ),
    "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
1061
1062
1063
1064
1065
1066
    "Ovis2_6ForCausalLM": _HfExamplesInfo(
        "AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
    ),
    "Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
        "AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
    ),
1067
1068
1069
1070
    "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
        "PaddlePaddle/PaddleOCR-VL",
        trust_remote_code=True,
    ),
1071
1072
1073
1074
1075
1076
1077
1078
    "PaliGemmaForConditionalGeneration": _HfExamplesInfo(
        "google/paligemma-3b-mix-224",
        extras={"v2": "google/paligemma2-3b-ft-docci-448"},
    ),
    "Phi3VForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-3-vision-128k-instruct",
        trust_remote_code=True,
        max_transformers_version="4.48",
1079
1080
1081
        transformers_version_reason={
            "hf": "HF model use deprecated imports which have been removed."
        },  # noqa: E501
1082
1083
        extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
    ),
1084
1085
1086
    "Phi4ForCausalLMV": _HfExamplesInfo(
        "microsoft/Phi-4-reasoning-vision-15B", trust_remote_code=True
    ),
1087
1088
1089
1090
1091
    "Phi4MMForCausalLM": _HfExamplesInfo(
        "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
    ),
    "PixtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Pixtral-12B-2409",
1092
1093
1094
1095
        extras={
            "mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
            "ministral-3": "mistralai/Ministral-3-3B-Instruct-2512",
        },
1096
1097
1098
1099
1100
1101
        tokenizer_mode="mistral",
    ),
    "QwenVLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen-VL",
        extras={"chat": "Qwen/Qwen-VL-Chat"},
        trust_remote_code=True,
1102
        max_transformers_version="4.53.3",
1103
1104
1105
        transformers_version_reason={
            "hf": "HF model uses deprecated imports which have been removed."
        },  # noqa: E501
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
    ),
    "Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2-Audio-7B-Instruct"
    ),
    "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"),
    "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-3B-Instruct",
        max_model_len=4096,
    ),
1116
    "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
    "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"),
    "Qwen3VLForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-4B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
    "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-VL-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
1128
    "Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
1129
        "Qwen/Qwen3.5-0.8B",
1130
1131
1132
        max_model_len=4096,
    ),
    "Qwen3_5MoeForConditionalGeneration": _HfExamplesInfo(
1133
        "Qwen/Qwen3.5-35B-A3B",
1134
1135
        max_model_len=4096,
    ),
1136
1137
1138
1139
1140
    "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
Roger Wang's avatar
Roger Wang committed
1141
    "Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
1142
        "Qwen/Qwen3-ASR-0.6B",
Roger Wang's avatar
Roger Wang committed
1143
1144
1145
        max_model_len=4096,
        min_transformers_version="4.57",
    ),
1146
    "Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
1147
        "Qwen/Qwen3-ASR-0.6B",
1148
1149
1150
1151
        max_model_len=4096,
        min_transformers_version="4.57",
        hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
    ),
1152
1153
1154
1155
1156
1157
    "Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
        "Qwen/Qwen3-ForcedAligner-0.6B",
        max_model_len=4096,
        min_transformers_version="4.57",
        hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
    ),
1158
1159
1160
1161
1162
    "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
    "SkyworkR1VChatModel": _HfExamplesInfo(
        "Skywork/Skywork-R1V-38B", trust_remote_code=True
    ),
    "SmolVLMForConditionalGeneration": _HfExamplesInfo(
1163
        "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
1164
1165
1166
1167
    ),
    "Step3VLForConditionalGeneration": _HfExamplesInfo(
        "stepfun-ai/step3", trust_remote_code=True
    ),
ltd0924's avatar
ltd0924 committed
1168
1169
1170
    "StepVLForConditionalGeneration": _HfExamplesInfo(
        "stepfun-ai/Step3-VL-10B", trust_remote_code=True
    ),
1171
1172
1173
1174
1175
1176
1177
    "UltravoxModel": _HfExamplesInfo(
        "fixie-ai/ultravox-v0_5-llama-3_2-1b",
        trust_remote_code=True,
    ),
    "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
    "Tarsier2ForConditionalGeneration": _HfExamplesInfo(
        "omni-research/Tarsier2-Recap-7b",
1178
1179
1180
1181
        hf_overrides={
            "architectures": ["Tarsier2ForConditionalGeneration"],
            "model_type": "tarsier2",
        },
1182
    ),
1183
1184
    "VoxtralForConditionalGeneration": _HfExamplesInfo(
        "mistralai/Voxtral-Mini-3B-2507",
1185
        tokenizer_mode="mistral",
1186
    ),
1187
    "VoxtralRealtimeGeneration": _HfExamplesInfo(
1188
1189
1190
        "mistralai/Voxtral-Mini-4B-Realtime-2602",
        enforce_eager=True,
        tokenizer_mode="mistral",
Patrick von Platen's avatar
Patrick von Platen committed
1191
    ),
1192
    # [Encoder-decoder]
1193
1194
    "CohereAsrForConditionalGeneration": _HfExamplesInfo(
        "CohereLabs/cohere-transcribe-03-2026",
Ekagra Ranjan's avatar
Ekagra Ranjan committed
1195
1196
1197
        trust_remote_code=True,
        is_available_online=False,  # TODO (ekagra): revert after asr release
    ),
1198
1199
1200
    "NemotronParseForConditionalGeneration": _HfExamplesInfo(
        "nvidia/NVIDIA-Nemotron-Parse-v1.1", trust_remote_code=True
    ),
1201
1202
1203
1204
    "WhisperForConditionalGeneration": _HfExamplesInfo(
        "openai/whisper-large-v3-turbo",
        extras={"v3": "openai/whisper-large-v3"},
    ),
1205
    # [Cross-encoder]
1206
    "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"),
1207
1208
}

1209

1210
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1211
    # [Medusa]
1212
1213
1214
    "MedusaModel": _HfExamplesInfo(
        "JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
    ),
1215
1216
    # Temporarily disabled.
    # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
1217
1218
1219
1220
    # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
    #     "JackFram/llama-160m",
    #     speculative_model="ibm-ai-platform/llama-160m-accelerator"
    # ),
1221
1222
1223
1224
1225
1226
1227
1228
    # [DFlash]
    "DFlashDraftModel": _HfExamplesInfo(
        "Qwen/Qwen3.5-4B",
        speculative_model="z-lab/Qwen3.5-4B-DFlash",
        use_original_num_layers=True,  # Need all layers since DFlash has >1 layer,
        max_model_len=8192,  # Reduce max len to ensure test runs in low-VRAM CI env
        max_num_seqs=32,
    ),
1229
    # [Eagle]
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
    "EagleDeepSeekMTPModel": _HfExamplesInfo(
        "eagle618/deepseek-v3-random",
        speculative_model="eagle618/eagle-deepseek-v3-random",
        trust_remote_code=True,
    ),
    "EagleLlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Meta-Llama-3-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B",
        tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
    ),
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
    "Eagle3DeepseekV2ForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True,
        speculative_model="AQ-MedAI/Kimi-K25-eagle3",
        tokenizer="moonshotai/Kimi-K2.5",
    ),
    "Eagle3DeepseekV3ForCausalLM": _HfExamplesInfo(
        "moonshotai/Kimi-K2.5",
        trust_remote_code=True,
        speculative_model="AQ-MedAI/Kimi-K25-eagle3",
        tokenizer="moonshotai/Kimi-K2.5",
    ),
1253
1254
1255
1256
1257
1258
1259
1260
    "Eagle3LlamaForCausalLM": _HfExamplesInfo(
        "meta-llama/Llama-3.1-8B-Instruct",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
        tokenizer="meta-llama/Llama-3.1-8B-Instruct",
        use_original_num_layers=True,
        max_model_len=10240,
    ),
1261
1262
1263
1264
1265
1266
    "Eagle3MiniMaxM2ForCausalLM": _HfExamplesInfo(
        "MiniMaxAI/MiniMax-M2",
        trust_remote_code=True,
        speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
        tokenizer="MiniMaxAI/MiniMax-M2",
    ),
1267
1268
1269
    "EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
        "mistralai/Mistral-Large-3-675B-Instruct-2512",
        speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
1270
        # TODO: revert once figuring out OOM in CI
1271
1272
        is_available_online=False,
    ),
1273
1274
1275
1276
1277
1278
1279
    "LlamaForCausalLMEagle3": _HfExamplesInfo(
        "Qwen/Qwen3-8B",
        trust_remote_code=True,
        speculative_model="AngelSlim/Qwen3-8B_eagle3",
        tokenizer="Qwen/Qwen3-8B",
        use_original_num_layers=True,
    ),
zhiweiz's avatar
zhiweiz committed
1280
1281
1282
1283
    "EagleLlama4ForCausalLM": _HfExamplesInfo(
        "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
        trust_remote_code=True,
        speculative_model="morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
1284
1285
1286
1287
1288
1289
        tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct",
    ),
    "EagleMiniCPMForCausalLM": _HfExamplesInfo(
        "openbmb/MiniCPM-1B-sft-bf16",
        trust_remote_code=True,
        speculative_model="openbmb/MiniCPM-2B-sft-bf16",
1290
        speculative_method="eagle",
1291
1292
        tokenizer="openbmb/MiniCPM-2B-sft-bf16",
    ),
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
    "Eagle3Qwen2_5vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen2.5-VL-7B-Instruct",
        speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl",
    ),
    "Eagle3Qwen3vlForCausalLM": _HfExamplesInfo(
        "Qwen/Qwen3-VL-8B-Instruct",
        speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
    ),
    # [MTP]
    "DeepSeekMTPModel": _HfExamplesInfo(
        "luccafong/deepseek_mtp_main_random",
        speculative_model="luccafong/deepseek_mtp_draft_random",
        trust_remote_code=True,
    ),
1307
1308
1309
1310
1311
    "ErnieMTPModel": _HfExamplesInfo(
        "baidu/ERNIE-4.5-21B-A3B-PT",
        trust_remote_code=True,
        speculative_model="baidu/ERNIE-4.5-21B-A3B-PT",
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
1312
1313
1314
    "ExaoneMoeMTP": _HfExamplesInfo(
        "LGAI-EXAONE/K-EXAONE-236B-A23B",
        speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
1315
        min_transformers_version="5.1.0",
1316
        enable_prefix_caching=False,
Kyungmin Lee's avatar
Kyungmin Lee committed
1317
    ),
Kyungmin Lee's avatar
Kyungmin Lee committed
1318
1319
1320
1321
1322
    "Exaone4_5_MTP": _HfExamplesInfo(
        "LGAI-EXAONE/EXAONE-4.5-33B",
        speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
        min_transformers_version="5.6.0",
    ),
1323
1324
1325
1326
    "ExtractHiddenStatesModel": _HfExamplesInfo(
        "Qwen/Qwen3-8B",
        speculative_method="extract_hidden_states",
    ),
1327
1328
1329
1330
    "Glm4MoeMTPModel": _HfExamplesInfo(
        "zai-org/GLM-4.5",
        speculative_model="zai-org/GLM-4.5",
    ),
1331
1332
1333
    "Glm4MoeLiteMTPModel": _HfExamplesInfo(
        "zai-org/GLM-4.7-Flash",
        speculative_model="zai-org/GLM-4.7-Flash",
1334
        min_transformers_version="5.0.0",
1335
1336
1337
1338
    ),
    "GlmOcrMTPModel": _HfExamplesInfo(
        "zai-org/GLM-OCR",
        speculative_model="zai-org/GLM-OCR",
1339
        is_available_online=False,
1340
        min_transformers_version="5.1.0",
1341
    ),
XuruiYang's avatar
XuruiYang committed
1342
1343
1344
    "LongCatFlashMTPModel": _HfExamplesInfo(
        "meituan-longcat/LongCat-Flash-Chat",
        trust_remote_code=True,
1345
1346
1347
1348
1349
1350
1351
        speculative_model="meituan-longcat/LongCat-Flash-Chat",
    ),
    "MiMoMTPModel": _HfExamplesInfo(
        "XiaomiMiMo/MiMo-7B-RL",
        trust_remote_code=True,
        speculative_model="XiaomiMiMo/MiMo-7B-RL",
    ),
1352
1353
1354
1355
    "NemotronHMTPModel": _HfExamplesInfo(
        "nvidia/Nemotron-Super-Placeholder",
        speculative_model="nvidia/Nemotron-Super-Placeholder",
        is_available_online=False,
1356
    ),
1357
1358
1359
1360
    "OpenPanguMTPModel": _HfExamplesInfo(
        "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
        trust_remote_code=True,
        is_available_online=False,
1361
    ),
1362
1363
1364
    "Qwen3NextMTP": _HfExamplesInfo(
        "Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
    ),
1365
1366
1367
1368
1369
1370
1371
1372
    "Qwen3_5MTP": _HfExamplesInfo(
        "Qwen/Qwen3.5-0.8B",
        speculative_model="Qwen/Qwen3.5-0.8B",
    ),
    "Qwen3_5MoeMTP": _HfExamplesInfo(
        "Qwen/Qwen3.5-35B-A3B",
        speculative_model="Qwen/Qwen3.5-35B-A3B",
    ),
csy0225's avatar
csy0225 committed
1373
1374
1375
    "Step3p5MTP": _HfExamplesInfo(
        "stepfun-ai/Step-3.5-Flash",
        speculative_model="stepfun-ai/Step-3.5-Flash",
1376
1377
        use_original_num_layers=True,
        # Initialize at least one MoE layer
1378
        hf_overrides={"num_hidden_layers": 4},
csy0225's avatar
csy0225 committed
1379
1380
        is_available_online=False,
    ),
1381
1382
}

1383
_TRANSFORMERS_BACKEND_MODELS = {
1384
    "TransformersEmbeddingModel": _HfExamplesInfo(
1385
        "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
1386
1387
1388
    ),
    "TransformersForSequenceClassification": _HfExamplesInfo(
        "papluca/xlm-roberta-base-language-detection",
1389
        min_transformers_version="5.0.0",
1390
1391
1392
1393
    ),
    "TransformersForCausalLM": _HfExamplesInfo(
        "hmellor/Ilama-3.2-1B", trust_remote_code=True
    ),
1394
    "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
1395
    "TransformersMoEForCausalLM": _HfExamplesInfo(
1396
        "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
1397
    ),
1398
    "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
1399
        "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
1400
1401
    ),
    "TransformersMoEEmbeddingModel": _HfExamplesInfo(
1402
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
1403
1404
    ),
    "TransformersMoEForSequenceClassification": _HfExamplesInfo(
1405
        "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
1406
    ),
1407
1408
1409
1410
    "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
    "TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
        "google/gemma-3-4b-it"
    ),
1411
1412
}

1413
1414
1415
_EXAMPLE_MODELS = {
    **_TEXT_GENERATION_EXAMPLE_MODELS,
    **_EMBEDDING_EXAMPLE_MODELS,
1416
1417
1418
    **_LATE_INTERACTION_EXAMPLE_MODELS,
    **_REWARD_EXAMPLE_MODELS,
    **_TOKEN_CLASSIFICATION_EXAMPLE_MODELS,
1419
    **_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS,
1420
1421
    **_MULTIMODAL_EXAMPLE_MODELS,
    **_SPECULATIVE_DECODING_EXAMPLE_MODELS,
1422
    **_TRANSFORMERS_BACKEND_MODELS,
1423
1424
1425
1426
1427
1428
1429
1430
1431
}


class HfExampleModels:
    def __init__(self, hf_models: Mapping[str, _HfExamplesInfo]) -> None:
        super().__init__()

        self.hf_models = hf_models

1432
    def get_supported_archs(self) -> Set[str]:
1433
1434
1435
        return self.hf_models.keys()

    def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
1436
1437
1438
        try:
            return self.hf_models[model_arch]
        except KeyError:
1439
1440
1441
            raise ValueError(
                f"No example model defined for {model_arch}; please update this file."
            ) from None
1442

1443
1444
1445
1446
1447
    def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
        for info in self.hf_models.values():
            if info.default == model_id:
                return info

1448
1449
1450
1451
1452
        # Fallback to extras
        for info in self.hf_models.values():
            if any(extra == model_id for extra in info.extras.values()):
                return info

1453
1454
1455
        raise ValueError(
            f"No example model defined for {model_id}; please update this file."
        )
1456

1457

Patrick von Platen's avatar
Patrick von Platen committed
1458
HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
1459
AUTO_EXAMPLE_MODELS = HfExampleModels(_AUTOMATIC_CONVERTED_MODELS)