test_config.py 19.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import os
5
from dataclasses import MISSING, Field, asdict, dataclass, field
6
from unittest.mock import patch
7

8
import pytest
9
import os
10

11
from vllm.compilation.backends import VllmBackend
12
from vllm.config import ModelConfig, PoolerConfig, VllmConfig, update_config
13
from vllm.config.load import LoadConfig
14
from vllm.config.utils import get_field
15
16
from vllm.model_executor.layers.pooler import PoolingType
from vllm.platforms import current_platform
17
from utils import models_path_prefix
18

19

20
21
22
23
24
25
26
27
28
29
30
31
def test_compile_config_repr_succeeds():
    # setup: VllmBackend mutates the config object
    config = VllmConfig()
    backend = VllmBackend(config)
    backend.configure_post_pass()

    # test that repr(config) succeeds
    val = repr(config)
    assert 'VllmConfig' in val
    assert 'inductor_passes' in val


32
33
34
35
36
@dataclass
class _TestConfigFields:
    a: int
    b: dict = field(default_factory=dict)
    c: str = "default"
37
38


39
def test_get_field():
40
    with pytest.raises(ValueError):
41
        get_field(_TestConfigFields, "a")
42

43
    b = get_field(_TestConfigFields, "b")
44
45
46
47
    assert isinstance(b, Field)
    assert b.default is MISSING
    assert b.default_factory is dict

48
    c = get_field(_TestConfigFields, "c")
49
50
51
52
53
    assert isinstance(c, Field)
    assert c.default == "default"
    assert c.default_factory is MISSING


54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
@dataclass
class _TestNestedConfig:
    a: _TestConfigFields = field(
        default_factory=lambda: _TestConfigFields(a=0))


def test_update_config():
    # Simple update
    config1 = _TestConfigFields(a=0)
    new_config1 = update_config(config1, {"a": 42})
    assert new_config1.a == 42
    # Nonexistent field
    with pytest.raises(AssertionError):
        new_config1 = update_config(config1, {"nonexistent": 1})
    # Nested update with dataclass
    config2 = _TestNestedConfig()
    new_inner_config = _TestConfigFields(a=1, c="new_value")
    new_config2 = update_config(config2, {"a": new_inner_config})
    assert new_config2.a == new_inner_config
    # Nested update with dict
    config3 = _TestNestedConfig()
    new_config3 = update_config(config3, {"a": {"c": "new_value"}})
    assert new_config3.a.c == "new_value"
    # Nested update with invalid type
    with pytest.raises(AssertionError):
        new_config3 = update_config(config3, {"a": "new_value"})


82
# Can remove once --task option is fully deprecated
83
@pytest.mark.parametrize(
84
85
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
86
    [
87
88
89
90
        (os.path.join(models_path_prefix, "distilbert/distilgpt2"), "generate", "none", "generate"),
        (os.path.join(models_path_prefix, "intfloat/multilingual-e5-small"), "pooling", "none", "embed"),
        (os.path.join(models_path_prefix, "jason9693/Qwen2.5-1.5B-apeach"), "pooling", "classify", "classify"),
        (os.path.join(models_path_prefix, "cross-encoder/ms-marco-MiniLM-L-6-v2"), "pooling", "none",
91
         "classify"),
92
93
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-Math-RM-72B"), "pooling", "none", "reward"),
        (os.path.join(models_path_prefix, "openai/whisper-small"), "generate", "none", "transcription"),
94
95
    ],
)
96
97
98
def test_auto_task(model_id, expected_runner_type, expected_convert_type,
                   expected_task):
    config = ModelConfig(model_id, task="auto")
99

100
    assert config.runner_type == expected_runner_type
101
    assert config.convert_type == expected_convert_type
102

103

104
105
106
107
108
# Can remove once --task option is fully deprecated
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
    [
zhuwenwen's avatar
zhuwenwen committed
109
110
111
112
        (os.path.join(models_path_prefix, "distilbert/distilgpt2"), "pooling", "embed", "embed"),
        (os.path.join(models_path_prefix, "intfloat/multilingual-e5-small"), "pooling", "embed", "embed"),
        (os.path.join(models_path_prefix, "jason9693/Qwen2.5-1.5B-apeach"), "pooling", "classify", "classify"),
        (os.path.join(models_path_prefix, "cross-encoder/ms-marco-MiniLM-L-6-v2"), "pooling", "classify",
113
         "classify"),
zhuwenwen's avatar
zhuwenwen committed
114
115
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-Math-RM-72B"), "pooling", "embed", "embed"),
        (os.path.join(models_path_prefix, "openai/whisper-small"), "pooling", "embed", "embed"),
116
117
118
119
120
    ],
)
def test_score_task(model_id, expected_runner_type, expected_convert_type,
                    expected_task):
    config = ModelConfig(model_id, task="score")
121

122
123
    assert config.runner_type == expected_runner_type
    assert config.convert_type == expected_convert_type
124

125
126

# Can remove once --task option is fully deprecated
127
@pytest.mark.parametrize(
128
129
    ("model_id", "expected_runner_type", "expected_convert_type",
     "expected_task"),
130
    [
131
        ("openai/whisper-small", "generate", "none", "transcription"),
132
133
    ],
)
134
135
136
def test_transcription_task(model_id, expected_runner_type,
                            expected_convert_type, expected_task):
    config = ModelConfig(model_id, task="transcription")
137

138
    assert config.runner_type == expected_runner_type
139
    assert config.convert_type == expected_convert_type
140
141


142
143
144
145
146
147
148
149
150
151
152
153
154
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type"),
    [
        ("distilbert/distilgpt2", "generate", "none"),
        ("intfloat/multilingual-e5-small", "pooling", "none"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"),
        ("openai/whisper-small", "generate", "none"),
    ],
)
def test_auto_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="auto")
155
156

    assert config.runner_type == expected_runner_type
157
    assert config.convert_type == expected_convert_type
158
159
160


@pytest.mark.parametrize(
161
    ("model_id", "expected_runner_type", "expected_convert_type"),
162
    [
163
164
165
166
167
168
        ("distilbert/distilgpt2", "pooling", "embed"),
        ("intfloat/multilingual-e5-small", "pooling", "none"),
        ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
        ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"),
        ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"),
        ("openai/whisper-small", "pooling", "embed"),
169
170
    ],
)
171
172
def test_pooling_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="pooling")
173
174

    assert config.runner_type == expected_runner_type
175
    assert config.convert_type == expected_convert_type
176
177


178
179
180
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_convert_type"),
    [
181
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-1.5B-Instruct"), "draft", "none"),
182
183
184
185
186
187
188
    ],
)
def test_draft_runner(model_id, expected_runner_type, expected_convert_type):
    config = ModelConfig(model_id, runner="draft")

    assert config.runner_type == expected_runner_type
    assert config.convert_type == expected_convert_type
189
190


191
MODEL_IDS_EXPECTED = [
192
193
194
    (os.path.join(models_path_prefix, "Qwen/Qwen1.5-7B"), 32768),
    (os.path.join(models_path_prefix, "mistralai/Mistral-7B-v0.1"), 4096),
    (os.path.join(models_path_prefix, "mistralai/Mistral-7B-Instruct-v0.2"), 32768),
195
196
197
198
199
200
]


@pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED)
def test_disable_sliding_window(model_id_expected):
    model_id, expected = model_id_expected
201
    model_config = ModelConfig(model_id, disable_sliding_window=True)
202
203
    assert model_config.max_model_len == expected

204

205
206
207
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config():
208
    model_id = os.path.join(models_path_prefix, "sentence-transformers/all-MiniLM-L12-v2")
209
    model_config = ModelConfig(model_id)
210

211
212
213
    assert model_config.pooler_config is not None
    assert model_config.pooler_config.normalize
    assert model_config.pooler_config.pooling_type == PoolingType.MEAN.name
214
215
216
217
218


@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config_from_args():
219
    model_id = os.path.join(models_path_prefix, "sentence-transformers/all-MiniLM-L12-v2")
220
221
    pooler_config = PoolerConfig(pooling_type="CLS", normalize=True)
    model_config = ModelConfig(model_id, pooler_config=pooler_config)
222

223
    assert asdict(model_config.pooler_config) == asdict(pooler_config)
224
225


226
227
228
229
230
231
232
233
234
235
236
237
238
239
@pytest.mark.parametrize(
    ("model_id", "default_pooling_type", "pooling_type"),
    [
        ("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "LAST", "LAST"),  # LLM
        ("intfloat/e5-small", "CLS", "MEAN"),  # BertModel
        ("Qwen/Qwen2.5-Math-RM-72B", "ALL", "ALL"),  # reward
        ("Qwen/Qwen2.5-Math-PRM-7B", "STEP", "STEP")  # step reward
    ])
def test_default_pooling_type(model_id, default_pooling_type, pooling_type):
    model_config = ModelConfig(model_id)
    assert model_config._model_info.default_pooling_type == default_pooling_type
    assert model_config.pooler_config.pooling_type == pooling_type


240
241
242
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_bert_tokenization_sentence_transformer_config():
243
244
    model_id = "BAAI/bge-base-en-v1.5"
    bge_model_config = ModelConfig(model_id)
245
246
247
248
249
250
251

    bert_bge_model_config = bge_model_config._get_encoder_config()

    assert bert_bge_model_config["max_seq_length"] == 512
    assert bert_bge_model_config["do_lower_case"]


252
def test_rope_customization():
253
    TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
254
    TEST_ROPE_THETA = 16_000_000.0
255
    LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
256

257
    llama_model_config = ModelConfig(os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"))
258
    assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
259
    assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
260
261
262
    assert llama_model_config.max_model_len == 8192

    llama_model_config = ModelConfig(
263
        os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"),
264
265
266
267
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
            "rope_theta": TEST_ROPE_THETA,
        },
268
269
270
    )
    assert getattr(llama_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
271
272
    assert getattr(llama_model_config.hf_config, "rope_theta",
                   None) == TEST_ROPE_THETA
273
274
    assert llama_model_config.max_model_len == 16384

275
    longchat_model_config = ModelConfig(os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"))
276
277
278
279
280
281
282
    # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
    assert all(
        longchat_model_config.hf_config.rope_scaling.get(key) == value
        for key, value in LONGCHAT_ROPE_SCALING.items())
    assert longchat_model_config.max_model_len == 16384

    longchat_model_config = ModelConfig(
283
        os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"),
284
285
286
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
        },
287
288
289
290
    )
    assert getattr(longchat_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
    assert longchat_model_config.max_model_len == 4096
291
292


293
294
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Encoder Decoder models not supported on ROCm.")
295
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
296
    (os.path.join(models_path_prefix, "facebook/opt-125m"), False),
297
    (os.path.join(models_path_prefix, "openai/whisper-tiny"), True),
298
    (os.path.join(models_path_prefix, "meta-llama/Llama-3.2-1B-Instruct"), False),
299
300
])
def test_is_encoder_decoder(model_id, is_encoder_decoder):
301
    config = ModelConfig(model_id)
302
303
304
305
306

    assert config.is_encoder_decoder == is_encoder_decoder


@pytest.mark.parametrize(("model_id", "uses_mrope"), [
307
308
    (os.path.join(models_path_prefix, "facebook/opt-125m"), False),
    (os.path.join(models_path_prefix, "Qwen/Qwen2-VL-2B-Instruct"), True),
309
310
])
def test_uses_mrope(model_id, uses_mrope):
311
    config = ModelConfig(model_id)
312
313

    assert config.uses_mrope == uses_mrope
314
315
316


def test_generation_config_loading():
317
    model_id = os.path.join(models_path_prefix, "Qwen/Qwen2.5-1.5B-Instruct")
318

319
    # When set generation_config to "vllm", the default generation config
320
    # will not be loaded.
321
    model_config = ModelConfig(model_id, generation_config="vllm")
322
323
324
325
    assert model_config.get_diff_sampling_param() == {}

    # When set generation_config to "auto", the default generation config
    # should be loaded.
326
    model_config = ModelConfig(model_id, generation_config="auto")
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349

    correct_generation_config = {
        "repetition_penalty": 1.1,
        "temperature": 0.7,
        "top_p": 0.8,
        "top_k": 20,
    }

    assert model_config.get_diff_sampling_param() == correct_generation_config

    # The generation config could be overridden by the user.
    override_generation_config = {"temperature": 0.5, "top_k": 5}

    model_config = ModelConfig(
        model_id,
        generation_config="auto",
        override_generation_config=override_generation_config)

    override_result = correct_generation_config.copy()
    override_result.update(override_generation_config)

    assert model_config.get_diff_sampling_param() == override_result

350
    # When generation_config is set to "vllm" and override_generation_config
351
352
353
    # is set, the override_generation_config should be used directly.
    model_config = ModelConfig(
        model_id,
354
        generation_config="vllm",
355
356
357
        override_generation_config=override_generation_config)

    assert model_config.get_diff_sampling_param() == override_generation_config
358
359
360
361
362
363
364
365
366
367
368
369
370


@pytest.mark.parametrize("pt_load_map_location", [
    "cuda",
    {
        "": "cuda"
    },
])
def test_load_config_pt_load_map_location(pt_load_map_location):
    load_config = LoadConfig(pt_load_map_location=pt_load_map_location)
    config = VllmConfig(load_config=load_config)

    assert config.load_config.pt_load_map_location == pt_load_map_location
371
372
373
374
375
376
377


@pytest.mark.parametrize(
    ("model_id", "max_model_len", "expected_max_len", "should_raise"), [
        ("BAAI/bge-reranker-base", None, 512, False),
        ("BAAI/bge-reranker-base", 256, 256, False),
        ("BAAI/bge-reranker-base", 513, 512, True),
378
379
        ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", None, 131072, False),
        ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", 131073, 131072, True),
380
381
382
383
    ])
def test_get_and_verify_max_len(model_id, max_model_len, expected_max_len,
                                should_raise):
    """Test get_and_verify_max_len with different configurations."""
384
    model_config = ModelConfig(model_id)
385
386
387
388
389
390
391

    if should_raise:
        with pytest.raises(ValueError):
            model_config.get_and_verify_max_len(max_model_len)
    else:
        actual_max_len = model_config.get_and_verify_max_len(max_model_len)
        assert actual_max_len == expected_max_len
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496


class MockConfig:
    """Simple mock object for testing maybe_pull_model_tokenizer_for_runai"""

    def __init__(self, model: str, tokenizer: str):
        self.model = model
        self.tokenizer = tokenizer
        self.model_weights = None


@pytest.mark.parametrize("s3_url", [
    "s3://example-bucket-1/model/",
    "s3://example-bucket-2/model/",
])
@patch('vllm.transformers_utils.runai_utils.ObjectStorageModel.pull_files')
def test_s3_url_model_tokenizer_paths(mock_pull_files, s3_url):
    """Test that S3 URLs create deterministic local directories for model and
    tokenizer."""
    # Mock pull_files to avoid actually downloading files during tests
    mock_pull_files.return_value = None

    # Create first mock and run the method
    config1 = MockConfig(model=s3_url, tokenizer=s3_url)
    ModelConfig.maybe_pull_model_tokenizer_for_runai(config1, s3_url, s3_url)

    # Check that model and tokenizer point to existing directories
    assert os.path.exists(
        config1.model), f"Model directory does not exist: {config1.model}"
    assert os.path.isdir(
        config1.model), f"Model path is not a directory: {config1.model}"
    assert os.path.exists(
        config1.tokenizer
    ), f"Tokenizer directory does not exist: {config1.tokenizer}"
    assert os.path.isdir(
        config1.tokenizer
    ), f"Tokenizer path is not a directory: {config1.tokenizer}"

    # Verify that the paths are different from the original S3 URL
    assert config1.model != s3_url, (
        "Model path should be converted to local directory")
    assert config1.tokenizer != s3_url, (
        "Tokenizer path should be converted to local directory")

    # Store the original paths
    created_model_dir = config1.model
    create_tokenizer_dir = config1.tokenizer

    # Create a new mock and run the method with the same S3 URL
    config2 = MockConfig(model=s3_url, tokenizer=s3_url)
    ModelConfig.maybe_pull_model_tokenizer_for_runai(config2, s3_url, s3_url)

    # Check that the new directories exist
    assert os.path.exists(
        config2.model), f"Model directory does not exist: {config2.model}"
    assert os.path.isdir(
        config2.model), f"Model path is not a directory: {config2.model}"
    assert os.path.exists(
        config2.tokenizer
    ), f"Tokenizer directory does not exist: {config2.tokenizer}"
    assert os.path.isdir(
        config2.tokenizer
    ), f"Tokenizer path is not a directory: {config2.tokenizer}"

    # Verify that the paths are deterministic (same as before)
    assert config2.model == created_model_dir, (
        f"Model paths are not deterministic. "
        f"Original: {created_model_dir}, New: {config2.model}")
    assert config2.tokenizer == create_tokenizer_dir, (
        f"Tokenizer paths are not deterministic. "
        f"Original: {create_tokenizer_dir}, New: {config2.tokenizer}")


@patch('vllm.transformers_utils.runai_utils.ObjectStorageModel.pull_files')
def test_s3_url_different_models_create_different_directories(mock_pull_files):
    """Test that different S3 URLs create different local directories."""
    # Mock pull_files to avoid actually downloading files during tests
    mock_pull_files.return_value = None

    s3_url1 = "s3://example-bucket-1/model/"
    s3_url2 = "s3://example-bucket-2/model/"

    # Create mocks with different S3 URLs and run the method
    config1 = MockConfig(model=s3_url1, tokenizer=s3_url1)
    ModelConfig.maybe_pull_model_tokenizer_for_runai(config1, s3_url1, s3_url1)

    config2 = MockConfig(model=s3_url2, tokenizer=s3_url2)
    ModelConfig.maybe_pull_model_tokenizer_for_runai(config2, s3_url2, s3_url2)

    # Verify that different URLs produce different directories
    assert config1.model != config2.model, (
        f"Different S3 URLs should create different model directories. "
        f"URL1 model: {config1.model}, URL2 model: {config2.model}")
    assert config1.tokenizer != config2.tokenizer, (
        f"Different S3 URLs should create different tokenizer directories. "
        f"URL1 tokenizer: {config1.tokenizer}, "
        f"URL2 tokenizer: {config2.tokenizer}")

    # Verify that both sets of directories exist
    assert os.path.exists(config1.model) and os.path.isdir(config1.model)
    assert os.path.exists(config1.tokenizer) and os.path.isdir(
        config1.tokenizer)
    assert os.path.exists(config2.model) and os.path.isdir(config2.model)
    assert os.path.exists(config2.tokenizer) and os.path.isdir(
        config2.tokenizer)