test_config.py 15.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from dataclasses import MISSING, Field, asdict, dataclass, field
5

6
import pytest
7
import os
8

9
from vllm.compilation.backends import VllmBackend
10
11
12
from vllm.config import (ModelConfig, PoolerConfig, VllmConfig, get_field,
                         update_config)
from vllm.config.load import LoadConfig
13
14
from vllm.model_executor.layers.pooler import PoolingType
from vllm.platforms import current_platform
15
from utils import models_path_prefix
16

17

18
19
20
21
22
23
24
25
26
27
28
29
def test_compile_config_repr_succeeds():
    # setup: VllmBackend mutates the config object
    config = VllmConfig()
    backend = VllmBackend(config)
    backend.configure_post_pass()

    # test that repr(config) succeeds
    val = repr(config)
    assert 'VllmConfig' in val
    assert 'inductor_passes' in val


30
31
32
33
34
@dataclass
class _TestConfigFields:
    a: int
    b: dict = field(default_factory=dict)
    c: str = "default"
35
36


37
def test_get_field():
38
    with pytest.raises(ValueError):
39
        get_field(_TestConfigFields, "a")
40

41
    b = get_field(_TestConfigFields, "b")
42
43
44
45
    assert isinstance(b, Field)
    assert b.default is MISSING
    assert b.default_factory is dict

46
    c = get_field(_TestConfigFields, "c")
47
48
49
50
51
    assert isinstance(c, Field)
    assert c.default == "default"
    assert c.default_factory is MISSING


52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
@dataclass
class _TestNestedConfig:
    a: _TestConfigFields = field(
        default_factory=lambda: _TestConfigFields(a=0))


def test_update_config():
    # Simple update
    config1 = _TestConfigFields(a=0)
    new_config1 = update_config(config1, {"a": 42})
    assert new_config1.a == 42
    # Nonexistent field
    with pytest.raises(AssertionError):
        new_config1 = update_config(config1, {"nonexistent": 1})
    # Nested update with dataclass
    config2 = _TestNestedConfig()
    new_inner_config = _TestConfigFields(a=1, c="new_value")
    new_config2 = update_config(config2, {"a": new_inner_config})
    assert new_config2.a == new_inner_config
    # Nested update with dict
    config3 = _TestNestedConfig()
    new_config3 = update_config(config3, {"a": {"c": "new_value"}})
    assert new_config3.a.c == "new_value"
    # Nested update with invalid type
    with pytest.raises(AssertionError):
        new_config3 = update_config(config3, {"a": "new_value"})


80
# Can remove once --task option is fully deprecated
81
@pytest.mark.parametrize(
82
83
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
84
    [
85
86
87
88
        (os.path.join(models_path_prefix, "distilbert/distilgpt2"), "generate", "none", "generate"),
        (os.path.join(models_path_prefix, "intfloat/multilingual-e5-small"), "pooling", "none", "embed"),
        (os.path.join(models_path_prefix, "jason9693/Qwen2.5-1.5B-apeach"), "pooling", "classify", "classify"),
        (os.path.join(models_path_prefix, "cross-encoder/ms-marco-MiniLM-L-6-v2"), "pooling", "none",
89
         "classify"),
90
91
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-Math-RM-72B"), "pooling", "none", "reward"),
        (os.path.join(models_path_prefix, "openai/whisper-small"), "generate", "none", "transcription"),
92
93
    ],
)
94
95
96
def test_auto_task(model_id, expected_runner_type, expected_convert_type,
                   expected_task):
    config = ModelConfig(model_id, task="auto")
97

98
    assert config.runner_type == expected_runner_type
99
100
    assert config.convert_type == expected_convert_type
    assert expected_task in config.supported_tasks
101

102

103
104
105
106
107
# Can remove once --task option is fully deprecated
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
    [
zhuwenwen's avatar
zhuwenwen committed
108
109
110
111
        (os.path.join(models_path_prefix, "distilbert/distilgpt2"), "pooling", "embed", "embed"),
        (os.path.join(models_path_prefix, "intfloat/multilingual-e5-small"), "pooling", "embed", "embed"),
        (os.path.join(models_path_prefix, "jason9693/Qwen2.5-1.5B-apeach"), "pooling", "classify", "classify"),
        (os.path.join(models_path_prefix, "cross-encoder/ms-marco-MiniLM-L-6-v2"), "pooling", "classify",
112
         "classify"),
zhuwenwen's avatar
zhuwenwen committed
113
114
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-Math-RM-72B"), "pooling", "embed", "embed"),
        (os.path.join(models_path_prefix, "openai/whisper-small"), "pooling", "embed", "embed"),
115
116
117
118
119
    ],
)
def test_score_task(model_id, expected_runner_type, expected_convert_type,
                    expected_task):
    config = ModelConfig(model_id, task="score")
120

121
122
123
    assert config.runner_type == expected_runner_type
    assert config.convert_type == expected_convert_type
    assert expected_task in config.supported_tasks
124

125
126

# Can remove once --task option is fully deprecated
127
@pytest.mark.parametrize(
128
129
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
130
    [
131
        ("openai/whisper-small", "generate", "none", "transcription"),
132
133
    ],
)
134
135
136
def test_transcription_task(model_id, expected_runner_type,
                            expected_convert_type, expected_task):
    config = ModelConfig(model_id, task="transcription")
137

138
    assert config.runner_type == expected_runner_type
139
140
    assert config.convert_type == expected_convert_type
    assert expected_task in config.supported_tasks
141
142


143
144
145
146
147
148
149
150
151
152
153
154
155
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type"),
    [
        ("distilbert/distilgpt2", "generate", "none"),
        ("intfloat/multilingual-e5-small", "pooling", "none"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"),
        ("openai/whisper-small", "generate", "none"),
    ],
)
def test_auto_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="auto")
156
157

    assert config.runner_type == expected_runner_type
158
    assert config.convert_type == expected_convert_type
159
160
161


@pytest.mark.parametrize(
162
    ("model_id", "expected_runner_type", "expected_convert_type"),
163
    [
164
165
166
167
168
169
        ("distilbert/distilgpt2", "pooling", "embed"),
        ("intfloat/multilingual-e5-small", "pooling", "none"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"),
        ("openai/whisper-small", "pooling", "embed"),
170
171
    ],
)
172
173
def test_pooling_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="pooling")
174
175

    assert config.runner_type == expected_runner_type
176
    assert config.convert_type == expected_convert_type
177
178


179
180
181
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type"),
    [
182
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-1.5B-Instruct"), "draft", "none"),
183
184
185
186
187
188
189
    ],
)
def test_draft_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="draft")

    assert config.runner_type == expected_runner_type
    assert config.convert_type == expected_convert_type
190
191


192
MODEL_IDS_EXPECTED = [
193
194
195
    (os.path.join(models_path_prefix, "Qwen/Qwen1.5-7B"), 32768),
    (os.path.join(models_path_prefix, "mistralai/Mistral-7B-v0.1"), 4096),
    (os.path.join(models_path_prefix, "mistralai/Mistral-7B-Instruct-v0.2"), 32768),
196
197
198
199
200
201
]


@pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED)
def test_disable_sliding_window(model_id_expected):
    model_id, expected = model_id_expected
202
    model_config = ModelConfig(model_id, disable_sliding_window=True)
203
204
    assert model_config.max_model_len == expected

205

206
207
208
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config():
209
    model_id = os.path.join(models_path_prefix, "sentence-transformers/all-MiniLM-L12-v2")
210
    model_config = ModelConfig(model_id)
211

212
    pooling_config = model_config._init_pooler_config()
213
    assert pooling_config is not None
214

215
216
    assert pooling_config.normalize
    assert pooling_config.pooling_type == PoolingType.MEAN.name
217
218
219
220
221


@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config_from_args():
222
    model_id = os.path.join(models_path_prefix, "sentence-transformers/all-MiniLM-L12-v2")
223
    model_config = ModelConfig(model_id)
224

225
226
    override_pooler_config = PoolerConfig(pooling_type='CLS', normalize=True)
    model_config.override_pooler_config = override_pooler_config
227

228
    pooling_config = model_config._init_pooler_config()
229
    assert pooling_config is not None
230
    assert asdict(pooling_config) == asdict(override_pooler_config)
231
232


233
234
235
236
237
238
239
240
241
242
243
244
245
246
@pytest.mark.parametrize(
    ("model_id", "default_pooling_type", "pooling_type"),
    [
        ("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "LAST", "LAST"),  # LLM
        ("intfloat/e5-small", "CLS", "MEAN"),  # BertModel
        ("Qwen/Qwen2.5-Math-RM-72B", "ALL", "ALL"),  # reward
        ("Qwen/Qwen2.5-Math-PRM-7B", "STEP", "STEP")  # step reward
    ])
def test_default_pooling_type(model_id, default_pooling_type, pooling_type):
    model_config = ModelConfig(model_id)
    assert model_config._model_info.default_pooling_type == default_pooling_type
    assert model_config.pooler_config.pooling_type == pooling_type


247
248
249
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_bert_tokenization_sentence_transformer_config():
250
251
    model_id = "BAAI/bge-base-en-v1.5"
    bge_model_config = ModelConfig(model_id)
252
253
254
255
256
257
258

    bert_bge_model_config = bge_model_config._get_encoder_config()

    assert bert_bge_model_config["max_seq_length"] == 512
    assert bert_bge_model_config["do_lower_case"]


259
def test_rope_customization():
260
    TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
261
    TEST_ROPE_THETA = 16_000_000.0
262
    LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
263

264
    llama_model_config = ModelConfig(os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"))
265
    assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
266
    assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
267
268
269
    assert llama_model_config.max_model_len == 8192

    llama_model_config = ModelConfig(
270
        os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"),
271
272
273
274
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
            "rope_theta": TEST_ROPE_THETA,
        },
275
276
277
    )
    assert getattr(llama_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
278
279
    assert getattr(llama_model_config.hf_config, "rope_theta",
                   None) == TEST_ROPE_THETA
280
281
    assert llama_model_config.max_model_len == 16384

282
    longchat_model_config = ModelConfig(os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"))
283
284
285
286
287
288
289
    # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
    assert all(
        longchat_model_config.hf_config.rope_scaling.get(key) == value
        for key, value in LONGCHAT_ROPE_SCALING.items())
    assert longchat_model_config.max_model_len == 16384

    longchat_model_config = ModelConfig(
290
        os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"),
291
292
293
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
        },
294
295
296
297
    )
    assert getattr(longchat_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
    assert longchat_model_config.max_model_len == 4096
298
299


300
301
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Encoder Decoder models not supported on ROCm.")
302
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
303
304
305
306
    (os.path.join(models_path_prefix, "facebook/opt-125m"), False),
    (os.path.join(models_path_prefix, "facebook/bart-base"), True),
    (os.path.join(models_path_prefix, "meta-llama/Llama-3.2-1B-Instruct"), False),
    (os.path.join(models_path_prefix, "meta-llama/Llama-3.2-11B-Vision"), True),
307
308
])
def test_is_encoder_decoder(model_id, is_encoder_decoder):
309
    config = ModelConfig(model_id)
310
311
312
313
314

    assert config.is_encoder_decoder == is_encoder_decoder


@pytest.mark.parametrize(("model_id", "uses_mrope"), [
315
316
    (os.path.join(models_path_prefix, "facebook/opt-125m"), False),
    (os.path.join(models_path_prefix, "Qwen/Qwen2-VL-2B-Instruct"), True),
317
318
])
def test_uses_mrope(model_id, uses_mrope):
319
    config = ModelConfig(model_id)
320
321

    assert config.uses_mrope == uses_mrope
322
323
324


def test_generation_config_loading():
325
    model_id = os.path.join(models_path_prefix, "Qwen/Qwen2.5-1.5B-Instruct")
326

327
    # When set generation_config to "vllm", the default generation config
328
    # will not be loaded.
329
    model_config = ModelConfig(model_id, generation_config="vllm")
330
331
332
333
    assert model_config.get_diff_sampling_param() == {}

    # When set generation_config to "auto", the default generation config
    # should be loaded.
334
    model_config = ModelConfig(model_id, generation_config="auto")
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357

    correct_generation_config = {
        "repetition_penalty": 1.1,
        "temperature": 0.7,
        "top_p": 0.8,
        "top_k": 20,
    }

    assert model_config.get_diff_sampling_param() == correct_generation_config

    # The generation config could be overridden by the user.
    override_generation_config = {"temperature": 0.5, "top_k": 5}

    model_config = ModelConfig(
        model_id,
        generation_config="auto",
        override_generation_config=override_generation_config)

    override_result = correct_generation_config.copy()
    override_result.update(override_generation_config)

    assert model_config.get_diff_sampling_param() == override_result

358
    # When generation_config is set to "vllm" and override_generation_config
359
360
361
    # is set, the override_generation_config should be used directly.
    model_config = ModelConfig(
        model_id,
362
        generation_config="vllm",
363
364
365
        override_generation_config=override_generation_config)

    assert model_config.get_diff_sampling_param() == override_generation_config
366
367
368
369
370
371
372
373
374
375
376
377
378


@pytest.mark.parametrize("pt_load_map_location", [
    "cuda",
    {
        "": "cuda"
    },
])
def test_load_config_pt_load_map_location(pt_load_map_location):
    load_config = LoadConfig(pt_load_map_location=pt_load_map_location)
    config = VllmConfig(load_config=load_config)

    assert config.load_config.pt_load_map_location == pt_load_map_location
379
380
381
382
383
384
385


@pytest.mark.parametrize(
    ("model_id", "max_model_len", "expected_max_len", "should_raise"), [
        ("BAAI/bge-reranker-base", None, 512, False),
        ("BAAI/bge-reranker-base", 256, 256, False),
        ("BAAI/bge-reranker-base", 513, 512, True),
386
387
        ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", None, 131072, False),
        ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", 131073, 131072, True),
388
389
390
391
    ])
def test_get_and_verify_max_len(model_id, max_model_len, expected_max_len,
                                should_raise):
    """Test get_and_verify_max_len with different configurations."""
392
    model_config = ModelConfig(model_id)
393
394
395
396
397
398
399

    if should_raise:
        with pytest.raises(ValueError):
            model_config.get_and_verify_max_len(max_model_len)
    else:
        actual_max_len = model_config.get_and_verify_max_len(max_model_len)
        assert actual_max_len == expected_max_len