test_config.py 14.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from dataclasses import MISSING, Field, asdict, dataclass, field
5
from typing import Literal, Union
6

7
import pytest
8
import os
9

10
from vllm.compilation.backends import VllmBackend
11
12
from vllm.config import (LoadConfig, ModelConfig, PoolerConfig, VllmConfig,
                         config, get_field)
13
14
from vllm.model_executor.layers.pooler import PoolingType
from vllm.platforms import current_platform
15
from utils import models_path_prefix
16

17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
class TestConfig1:
    pass


@dataclass
class TestConfig2:
    a: int
    """docstring"""


@dataclass
class TestConfig3:
    a: int = 1


@dataclass
class TestConfig4:
    a: Union[Literal[1], Literal[2]] = 1
    """docstring"""


@pytest.mark.parametrize(("test_config", "expected_error"), [
    (TestConfig1, "must be a dataclass"),
    (TestConfig2, "must have a default"),
    (TestConfig3, "must have a docstring"),
    (TestConfig4, "must use a single Literal"),
])
def test_config(test_config, expected_error):
    with pytest.raises(Exception, match=expected_error):
        config(test_config)


50
51
52
53
54
55
56
57
58
59
60
61
def test_compile_config_repr_succeeds():
    # setup: VllmBackend mutates the config object
    config = VllmConfig()
    backend = VllmBackend(config)
    backend.configure_post_pass()

    # test that repr(config) succeeds
    val = repr(config)
    assert 'VllmConfig' in val
    assert 'inductor_passes' in val


62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def test_get_field():

    @dataclass
    class TestConfig:
        a: int
        b: dict = field(default_factory=dict)
        c: str = "default"

    with pytest.raises(ValueError):
        get_field(TestConfig, "a")

    b = get_field(TestConfig, "b")
    assert isinstance(b, Field)
    assert b.default is MISSING
    assert b.default_factory is dict

    c = get_field(TestConfig, "c")
    assert isinstance(c, Field)
    assert c.default == "default"
    assert c.default_factory is MISSING


84
85
86
@pytest.mark.parametrize(
    ("model_id", "expected_runner_type", "expected_task"),
    [
zhuwenwen's avatar
zhuwenwen committed
87
88
89
90
        (os.path.join(models_path_prefix, "distilbert/distilgpt2"), "generate", "generate"),
        (os.path.join(models_path_prefix, "intfloat/multilingual-e5-small"), "pooling", "embed"),
        (os.path.join(models_path_prefix, "jason9693/Qwen2.5-1.5B-apeach"), "pooling", "classify"),
        (os.path.join(models_path_prefix, "cross-encoder/ms-marco-MiniLM-L-6-v2"), "pooling", "score"),
zhuwenwen's avatar
zhuwenwen committed
91
        (os.path.join(models_path_prefix, "Qwen/Qwen2.5-Math-RM-72B"), "pooling", "reward"),
zhuwenwen's avatar
zhuwenwen committed
92
        (os.path.join(models_path_prefix, "openai/whisper-small"), "transcription", "transcription"),
93
94
95
    ],
)
def test_auto_task(model_id, expected_runner_type, expected_task):
96
97
98
99
100
101
102
103
104
105
    config = ModelConfig(
        model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
    )

106
    assert config.runner_type == expected_runner_type
107
108
109
110
    assert config.task == expected_task


@pytest.mark.parametrize(("model_id", "bad_task"), [
zhuwenwen's avatar
zhuwenwen committed
111
    (os.path.join(models_path_prefix, "Qwen/Qwen2.5-Math-RM-72B"), "generate"),
112
113
114
115
116
117
118
119
120
121
122
123
124
125
])
def test_incorrect_task(model_id, bad_task):
    with pytest.raises(ValueError, match=r"does not support the .* task"):
        ModelConfig(
            model_id,
            task=bad_task,
            tokenizer=model_id,
            tokenizer_mode="auto",
            trust_remote_code=False,
            seed=0,
            dtype="float16",
        )


126
MODEL_IDS_EXPECTED = [
127
128
129
    (os.path.join(models_path_prefix, "Qwen/Qwen1.5-7B"), 32768),
    (os.path.join(models_path_prefix, "mistralai/Mistral-7B-v0.1"), 4096),
    (os.path.join(models_path_prefix, "mistralai/Mistral-7B-Instruct-v0.2"), 32768),
130
131
132
133
134
135
136
137
]


@pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED)
def test_disable_sliding_window(model_id_expected):
    model_id, expected = model_id_expected
    model_config = ModelConfig(
        model_id,
138
139
        task="auto",
        tokenizer=model_id,
140
141
142
143
144
145
146
147
148
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
        revision=None,
        disable_sliding_window=True,
    )
    assert model_config.max_model_len == expected

149
150
151
152
153
154
155

def test_get_sliding_window():
    TEST_SLIDING_WINDOW = 4096
    # Test that the sliding window is correctly computed.
    # For Qwen1.5/Qwen2, get_sliding_window() should be None
    # when use_sliding_window is False.
    qwen2_model_config = ModelConfig(
156
        os.path.join(models_path_prefix, "Qwen/Qwen1.5-7B"),
157
        task="auto",
zhuwenwen's avatar
zhuwenwen committed
158
        tokenizer=os.path.join(models_path_prefix, "Qwen/Qwen1.5-7B"),
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
        revision=None,
    )

    qwen2_model_config.hf_config.use_sliding_window = False
    qwen2_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
    assert qwen2_model_config.get_sliding_window() is None

    qwen2_model_config.hf_config.use_sliding_window = True
    assert qwen2_model_config.get_sliding_window() == TEST_SLIDING_WINDOW

    mistral_model_config = ModelConfig(
174
        os.path.join(models_path_prefix, "mistralai/Mistral-7B-v0.1"),
175
        task="auto",
zhuwenwen's avatar
zhuwenwen committed
176
        tokenizer=os.path.join(models_path_prefix, "mistralai/Mistral-7B-v0.1"),
177
178
179
180
181
182
183
184
185
186
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
        revision=None,
    )
    mistral_model_config.hf_config.sliding_window = None
    assert mistral_model_config.get_sliding_window() is None

    mistral_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW
187
188
189
    assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW


190
191
192
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config():
193
    model_id = os.path.join(models_path_prefix, "sentence-transformers/all-MiniLM-L12-v2")
194
    model_config = ModelConfig(
195
196
197
198
199
200
201
202
203
204
        model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
        revision=None,
    )

205
    pooling_config = model_config._init_pooler_config()
206
    assert pooling_config is not None
207

208
209
    assert pooling_config.normalize
    assert pooling_config.pooling_type == PoolingType.MEAN.name
210
211
212
213
214


@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_pooling_config_from_args():
215
    model_id = os.path.join(models_path_prefix, "sentence-transformers/all-MiniLM-L12-v2")
216
217
218
219
220
221
222
223
224
    model_config = ModelConfig(model_id,
                               task="auto",
                               tokenizer=model_id,
                               tokenizer_mode="auto",
                               trust_remote_code=False,
                               seed=0,
                               dtype="float16",
                               revision=None)

225
226
    override_pooler_config = PoolerConfig(pooling_type='CLS', normalize=True)
    model_config.override_pooler_config = override_pooler_config
227

228
    pooling_config = model_config._init_pooler_config()
229
    assert pooling_config is not None
230
    assert asdict(pooling_config) == asdict(override_pooler_config)
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252


@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Xformers backend is not supported on ROCm.")
def test_get_bert_tokenization_sentence_transformer_config():
    bge_model_config = ModelConfig(
        model="BAAI/bge-base-en-v1.5",
        task="auto",
        tokenizer="BAAI/bge-base-en-v1.5",
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
        revision=None,
    )

    bert_bge_model_config = bge_model_config._get_encoder_config()

    assert bert_bge_model_config["max_seq_length"] == 512
    assert bert_bge_model_config["do_lower_case"]


253
def test_rope_customization():
254
    TEST_ROPE_SCALING = {"rope_type": "dynamic", "factor": 2.0}
255
    TEST_ROPE_THETA = 16_000_000.0
256
    LONGCHAT_ROPE_SCALING = {"rope_type": "linear", "factor": 8.0}
257
258

    llama_model_config = ModelConfig(
259
        os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"),
260
        task="auto",
zhuwenwen's avatar
zhuwenwen committed
261
        tokenizer=os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"),
262
263
264
265
266
267
        tokenizer_mode="auto",
        trust_remote_code=False,
        dtype="float16",
        seed=0,
    )
    assert getattr(llama_model_config.hf_config, "rope_scaling", None) is None
268
    assert getattr(llama_model_config.hf_config, "rope_theta", None) == 500_000
269
270
271
    assert llama_model_config.max_model_len == 8192

    llama_model_config = ModelConfig(
272
        os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"),
273
        task="auto",
zhuwenwen's avatar
zhuwenwen committed
274
        tokenizer=os.path.join(models_path_prefix, "meta-llama/Meta-Llama-3-8B-Instruct"),
275
276
277
278
        tokenizer_mode="auto",
        trust_remote_code=False,
        dtype="float16",
        seed=0,
279
280
281
282
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
            "rope_theta": TEST_ROPE_THETA,
        },
283
284
285
    )
    assert getattr(llama_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
286
287
    assert getattr(llama_model_config.hf_config, "rope_theta",
                   None) == TEST_ROPE_THETA
288
289
    assert llama_model_config.max_model_len == 16384

290
    longchat_model_config = ModelConfig(
291
        os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"),
292
        task="auto",
zhuwenwen's avatar
zhuwenwen committed
293
        tokenizer=os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"),
294
295
296
297
298
299
300
301
302
303
304
305
        tokenizer_mode="auto",
        trust_remote_code=False,
        dtype="float16",
        seed=0,
    )
    # Check if LONGCHAT_ROPE_SCALING entries are in longchat_model_config
    assert all(
        longchat_model_config.hf_config.rope_scaling.get(key) == value
        for key, value in LONGCHAT_ROPE_SCALING.items())
    assert longchat_model_config.max_model_len == 16384

    longchat_model_config = ModelConfig(
306
        os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"),
307
        task="auto",
zhuwenwen's avatar
zhuwenwen committed
308
        tokenizer=os.path.join(models_path_prefix, "lmsys/longchat-13b-16k"),
309
310
311
312
        tokenizer_mode="auto",
        trust_remote_code=False,
        dtype="float16",
        seed=0,
313
314
315
        hf_overrides={
            "rope_scaling": TEST_ROPE_SCALING,
        },
316
317
318
319
    )
    assert getattr(longchat_model_config.hf_config, "rope_scaling",
                   None) == TEST_ROPE_SCALING
    assert longchat_model_config.max_model_len == 4096
320
321


322
323
@pytest.mark.skipif(current_platform.is_rocm(),
                    reason="Encoder Decoder models not supported on ROCm.")
324
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
325
326
327
328
    (os.path.join(models_path_prefix, "facebook/opt-125m"), False),
    (os.path.join(models_path_prefix, "facebook/bart-base"), True),
    (os.path.join(models_path_prefix, "meta-llama/Llama-3.2-1B-Instruct"), False),
    (os.path.join(models_path_prefix, "meta-llama/Llama-3.2-11B-Vision"), True),
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
])
def test_is_encoder_decoder(model_id, is_encoder_decoder):
    config = ModelConfig(
        model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        dtype="float16",
        seed=0,
    )

    assert config.is_encoder_decoder == is_encoder_decoder


@pytest.mark.parametrize(("model_id", "uses_mrope"), [
345
346
    (os.path.join(models_path_prefix, "facebook/opt-125m"), False),
    (os.path.join(models_path_prefix, "Qwen/Qwen2-VL-2B-Instruct"), True),
347
348
349
350
351
352
353
354
355
356
357
358
359
])
def test_uses_mrope(model_id, uses_mrope):
    config = ModelConfig(
        model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        dtype="float16",
        seed=0,
    )

    assert config.uses_mrope == uses_mrope
360
361
362


def test_generation_config_loading():
363
    model_id = os.path.join(models_path_prefix, "Qwen/Qwen2.5-1.5B-Instruct")
364

365
    # When set generation_config to "vllm", the default generation config
366
367
368
369
370
371
372
373
    # will not be loaded.
    model_config = ModelConfig(model_id,
                               task="auto",
                               tokenizer=model_id,
                               tokenizer_mode="auto",
                               trust_remote_code=False,
                               seed=0,
                               dtype="float16",
374
                               generation_config="vllm")
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
    assert model_config.get_diff_sampling_param() == {}

    # When set generation_config to "auto", the default generation config
    # should be loaded.
    model_config = ModelConfig(model_id,
                               task="auto",
                               tokenizer=model_id,
                               tokenizer_mode="auto",
                               trust_remote_code=False,
                               seed=0,
                               dtype="float16",
                               generation_config="auto")

    correct_generation_config = {
        "repetition_penalty": 1.1,
        "temperature": 0.7,
        "top_p": 0.8,
        "top_k": 20,
    }

    assert model_config.get_diff_sampling_param() == correct_generation_config

    # The generation config could be overridden by the user.
    override_generation_config = {"temperature": 0.5, "top_k": 5}

    model_config = ModelConfig(
        model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
        generation_config="auto",
        override_generation_config=override_generation_config)

    override_result = correct_generation_config.copy()
    override_result.update(override_generation_config)

    assert model_config.get_diff_sampling_param() == override_result

416
    # When generation_config is set to "vllm" and override_generation_config
417
418
419
420
421
422
423
424
425
    # is set, the override_generation_config should be used directly.
    model_config = ModelConfig(
        model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="float16",
426
        generation_config="vllm",
427
428
429
        override_generation_config=override_generation_config)

    assert model_config.get_diff_sampling_param() == override_generation_config
430
431
432
433
434
435
436
437
438
439
440
441
442


@pytest.mark.parametrize("pt_load_map_location", [
    "cuda",
    {
        "": "cuda"
    },
])
def test_load_config_pt_load_map_location(pt_load_map_location):
    load_config = LoadConfig(pt_load_map_location=pt_load_map_location)
    config = VllmConfig(load_config=load_config)

    assert config.load_config.pt_load_map_location == pt_load_map_location