test_config.py 14 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from dataclasses import MISSING, Field, asdict, dataclass, field
5

6
7
import pytest

8
from vllm.compilation.backends import VllmBackend
9
10
11
from vllm.config import (ModelConfig, PoolerConfig, VllmConfig, get_field,
                         update_config)
from vllm.config.load import LoadConfig
12
13
from vllm.model_executor.layers.pooler import PoolingType
from vllm.platforms import current_platform
14

15

16
17
18
19
20
21
22
23
24
25
26
27
def test_compile_config_repr_succeeds():
    # setup: VllmBackend mutates the config object
    config = VllmConfig()
    backend = VllmBackend(config)
    backend.configure_post_pass()

    # test that repr(config) succeeds
    val = repr(config)
    assert 'VllmConfig' in val
    assert 'inductor_passes' in val


28
29
30
31
32
@dataclass
class _TestConfigFields:
    a: int
    b: dict = field(default_factory=dict)
    c: str = "default"
33
34


35
def test_get_field():
36
    with pytest.raises(ValueError):
37
        get_field(_TestConfigFields, "a")
38

39
    b = get_field(_TestConfigFields, "b")
40
41
42
43
    assert isinstance(b, Field)
    assert b.default is MISSING
    assert b.default_factory is dict

44
    c = get_field(_TestConfigFields, "c")
45
46
47
48
49
    assert isinstance(c, Field)
    assert c.default == "default"
    assert c.default_factory is MISSING


50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
@dataclass
class _TestNestedConfig:
    a: _TestConfigFields = field(
        default_factory=lambda: _TestConfigFields(a=0))


def test_update_config():
    # Simple update
    config1 = _TestConfigFields(a=0)
    new_config1 = update_config(config1, {"a": 42})
    assert new_config1.a == 42
    # Nonexistent field
    with pytest.raises(AssertionError):
        new_config1 = update_config(config1, {"nonexistent": 1})
    # Nested update with dataclass
    config2 = _TestNestedConfig()
    new_inner_config = _TestConfigFields(a=1, c="new_value")
    new_config2 = update_config(config2, {"a": new_inner_config})
    assert new_config2.a == new_inner_config
    # Nested update with dict
    config3 = _TestNestedConfig()
    new_config3 = update_config(config3, {"a": {"c": "new_value"}})
    assert new_config3.a.c == "new_value"
    # Nested update with invalid type
    with pytest.raises(AssertionError):
        new_config3 = update_config(config3, {"a": "new_value"})


78
# Can remove once --task option is fully deprecated
79
@pytest.mark.parametrize(
80
81
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
82
    [
83
84
85
86
87
88
89
        ("distilbert/distilgpt2", "generate", "none", "generate"),
        ("intfloat/multilingual-e5-small", "pooling", "none", "embed"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none",
         "classify"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none", "reward"),
        ("openai/whisper-small", "generate", "none", "transcription"),
90
91
    ],
)
92
93
94
def test_auto_task(model_id, expected_runner_type, expected_convert_type,
                   expected_task):
    config = ModelConfig(model_id, task="auto")
95
96

    assert config.runner_type == expected_runner_type
97
98
    assert config.convert_type == expected_convert_type
    assert expected_task in config.supported_tasks
99

100

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
# Can remove once --task option is fully deprecated
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
    [
        ("distilbert/distilgpt2", "pooling", "embed", "embed"),
        ("intfloat/multilingual-e5-small", "pooling", "embed", "embed"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "classify",
         "classify"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "embed", "embed"),
        ("openai/whisper-small", "pooling", "embed", "embed"),
    ],
)
def test_score_task(model_id, expected_runner_type, expected_convert_type,
                    expected_task):
    config = ModelConfig(model_id, task="score")
118

119
120
121
122
123
124
    assert config.runner_type == expected_runner_type
    assert config.convert_type == expected_convert_type
    assert expected_task in config.supported_tasks


# Can remove once --task option is fully deprecated
125
@pytest.mark.parametrize(
126
127
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
128
    [
129
        ("openai/whisper-small", "generate", "none", "transcription"),
130
131
    ],
)
132
133
134
def test_transcription_task(model_id, expected_runner_type,
                            expected_convert_type, expected_task):
    config = ModelConfig(model_id, task="transcription")
135

136
    assert config.runner_type == expected_runner_type
137
138
    assert config.convert_type == expected_convert_type
    assert expected_task in config.supported_tasks
139
140


141
142
143
144
145
146
147
148
149
150
151
152
153
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type"),
    [
        ("distilbert/distilgpt2", "generate", "none"),
        ("intfloat/multilingual-e5-small", "pooling", "none"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"),
        ("openai/whisper-small", "generate", "none"),
    ],
)
def test_auto_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="auto")
154
155

    assert config.runner_type == expected_runner_type
156
    assert config.convert_type == expected_convert_type
157
158
159


@pytest.mark.parametrize(
160
    ("model_id", "expected_runner_type", "expected_convert_type"),
161
    [
162
163
164
165
166
167
        ("distilbert/distilgpt2", "pooling", "embed"),
        ("intfloat/multilingual-e5-small", "pooling", "none"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"),
        ("openai/whisper-small", "pooling", "embed"),
168
169
    ],
)
170
171
def test_pooling_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="pooling")
172
173

    assert config.runner_type == expected_runner_type
174
    assert config.convert_type == expected_convert_type
175
176


177
178
179
180
181
182
183
184
185
186
187
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type"),
    [
        ("Qwen/Qwen2.5-1.5B-Instruct", "draft", "none"),
    ],
)
def test_draft_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="draft")

    assert config.runner_type == expected_runner_type
    assert config.convert_type == expected_convert_type
188
189


190
191
192
193
194
195
196
197
198
199
MODEL_IDS_EXPECTED = [
    ("Qwen/Qwen1.5-7B", 32768),
    ("mistralai/Mistral-7B-v0.1", 4096),
    ("mistralai/Mistral-7B-Instruct-v0.2", 32768),
]


@pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED)
def test_disable_sliding_window(model_id_expected):
    model_id, expected = model_id_expected
200
    model_config = ModelConfig(model_id, disable_sliding_window=True)
201
202
    assert model_config.max_model_len == expected

203

204
205
206
207
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config():
    model_id = "sentence-transformers/all-MiniLM-L12-v2"
208
    model_config = ModelConfig(model_id)
209

210
    pooling_config = model_config._init_pooler_config()
211
    assert pooling_config is not None
212

213
214
    assert pooling_config.normalize
    assert pooling_config.pooling_type == PoolingType.MEAN.name
215
216
217
218
219
220


@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config_from_args():
    model_id = "sentence-transformers/all-MiniLM-L12-v2"
221
    model_config = ModelConfig(model_id)
222

223
224
    override_pooler_config = PoolerConfig(pooling_type='CLS', normalize=True)
    model_config.override_pooler_config = override_pooler_config
225

226
    pooling_config = model_config._init_pooler_config()
227
    assert pooling_config is not None
228
    assert asdict(pooling_config) == asdict(override_pooler_config)
229
230


231
232
233
234
235
236
237
238
239
240
241
242
243
244
@pytest.mark.parametrize(
    ("model_id", "default_pooling_type", "pooling_type"),
    [
        ("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "LAST", "LAST"),  # LLM
        ("intfloat/e5-small", "CLS", "MEAN"),  # BertModel
        ("Qwen/Qwen2.5-Math-RM-72B", "ALL", "ALL"),  # reward
        ("Qwen/Qwen2.5-Math-PRM-7B", "STEP", "STEP")  # step reward
    ])
def test_default_pooling_type(model_id, default_pooling_type, pooling_type):
    model_config = ModelConfig(model_id)
    assert model_config._model_info.default_pooling_type == default_pooling_type
    assert model_config.pooler_config.pooling_type == pooling_type


245
246
247
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_bert_tokenization_sentence_transformer_config():
248
249
    model_id = "BAAI/bge-base-en-v1.5"
    bge_model_config = ModelConfig(model_id)
250
251
252
253
254
255
256

    bert_bge_model_config = bge_model_config._get_encoder_config()

    assert bert_bge_model_config["max_seq_length"] == 512
    assert bert_bge_model_config["do_lower_case"]


257
def test_rope_customization():
258
    TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
259
    TEST_ROPE_THETA = 16_000_000.0
260
    LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
261

262
    llama_model_config = ModelConfig("meta-llama/Meta-Llama-3-8B-Instruct")
263
    assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
264
    assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
265
266
267
268
    assert llama_model_config.max_model_len == 8192

    llama_model_config = ModelConfig(
        "meta-llama/Meta-Llama-3-8B-Instruct",
269
270
271
272
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
            "rope_theta": TEST_ROPE_THETA,
        },
273
274
275
    )
    assert getattr(llama_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
276
277
    assert getattr(llama_model_config.hf_config, "rope_theta",
                   None) == TEST_ROPE_THETA
278
279
    assert llama_model_config.max_model_len == 16384

280
    longchat_model_config = ModelConfig("lmsys/longchat-13b-16k")
281
282
283
284
285
286
287
288
    # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
    assert all(
        longchat_model_config.hf_config.rope_scaling.get(key) == value
        for key, value in LONGCHAT_ROPE_SCALING.items())
    assert longchat_model_config.max_model_len == 16384

    longchat_model_config = ModelConfig(
        "lmsys/longchat-13b-16k",
289
290
291
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
        },
292
293
294
295
    )
    assert getattr(longchat_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
    assert longchat_model_config.max_model_len == 4096
296
297


298
299
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Encoder Decoder models not supported on ROCm.")
300
301
302
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
    ("facebook/opt-125m", False),
    ("facebook/bart-base", True),
303
    ("meta-llama/Llama-3.2-1B-Instruct", False),
304
305
306
    ("meta-llama/Llama-3.2-11B-Vision", True),
])
def test_is_encoder_decoder(model_id, is_encoder_decoder):
307
    config = ModelConfig(model_id)
308
309
310
311
312
313
314
315
316

    assert config.is_encoder_decoder == is_encoder_decoder


@pytest.mark.parametrize(("model_id", "uses_mrope"), [
    ("facebook/opt-125m", False),
    ("Qwen/Qwen2-VL-2B-Instruct", True),
])
def test_uses_mrope(model_id, uses_mrope):
317
    config = ModelConfig(model_id)
318
319

    assert config.uses_mrope == uses_mrope
320
321
322
323
324


def test_generation_config_loading():
    model_id = "Qwen/Qwen2.5-1.5B-Instruct"

325
    # When set generation_config to "vllm", the default generation config
326
    # will not be loaded.
327
    model_config = ModelConfig(model_id, generation_config="vllm")
328
329
330
331
    assert model_config.get_diff_sampling_param() == {}

    # When set generation_config to "auto", the default generation config
    # should be loaded.
332
    model_config = ModelConfig(model_id, generation_config="auto")
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355

    correct_generation_config = {
        "repetition_penalty": 1.1,
        "temperature": 0.7,
        "top_p": 0.8,
        "top_k": 20,
    }

    assert model_config.get_diff_sampling_param() == correct_generation_config

    # The generation config could be overridden by the user.
    override_generation_config = {"temperature": 0.5, "top_k": 5}

    model_config = ModelConfig(
        model_id,
        generation_config="auto",
        override_generation_config=override_generation_config)

    override_result = correct_generation_config.copy()
    override_result.update(override_generation_config)

    assert model_config.get_diff_sampling_param() == override_result

356
    # When generation_config is set to "vllm" and override_generation_config
357
358
359
    # is set, the override_generation_config should be used directly.
    model_config = ModelConfig(
        model_id,
360
        generation_config="vllm",
361
362
363
        override_generation_config=override_generation_config)

    assert model_config.get_diff_sampling_param() == override_generation_config
364
365
366
367
368
369
370
371
372
373
374
375
376


@pytest.mark.parametrize("pt_load_map_location", [
    "cuda",
    {
        "": "cuda"
    },
])
def test_load_config_pt_load_map_location(pt_load_map_location):
    load_config = LoadConfig(pt_load_map_location=pt_load_map_location)
    config = VllmConfig(load_config=load_config)

    assert config.load_config.pt_load_map_location == pt_load_map_location
377
378
379
380
381
382
383


@pytest.mark.parametrize(
    ("model_id", "max_model_len", "expected_max_len", "should_raise"), [
        ("BAAI/bge-reranker-base", None, 512, False),
        ("BAAI/bge-reranker-base", 256, 256, False),
        ("BAAI/bge-reranker-base", 513, 512, True),
384
385
        ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", None, 131072, False),
        ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", 131073, 131072, True),
386
387
388
389
    ])
def test_get_and_verify_max_len(model_id, max_model_len, expected_max_len,
                                should_raise):
    """Test get_and_verify_max_len with different configurations."""
390
    model_config = ModelConfig(model_id)
391
392
393
394
395
396
397

    if should_raise:
        with pytest.raises(ValueError):
            model_config.get_and_verify_max_len(max_model_len)
    else:
        actual_max_len = model_config.get_and_verify_max_len(max_model_len)
        assert actual_max_len == expected_max_len