test_sglang.py 16 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import dataclasses
5
6
import logging
import os
7
from dataclasses import dataclass, field
8
9
10

import pytest

Alec's avatar
Alec committed
11
12
from tests.serve.common import (
    SERVE_TEST_DIR,
13
    WORKSPACE_DIR,
Alec's avatar
Alec committed
14
15
16
    params_with_model_mark,
    run_serve_deployment,
)
17
from tests.utils.constants import DefaultPort
18
from tests.utils.engine_process import EngineConfig
19
from tests.utils.payload_builder import (
20
21
    anthropic_messages_payload_default,
    anthropic_messages_stream_payload_default,
22
23
24
    chat_payload,
    chat_payload_default,
    completion_payload_default,
25
26
    embedding_payload,
    embedding_payload_default,
27
    metric_payload_default,
28
29
    responses_payload_default,
    responses_stream_payload_default,
30
)
31
32
33
34
35

logger = logging.getLogger(__name__)


@dataclass
36
class SGLangConfig(EngineConfig):
37
38
    """Configuration for SGLang test scenarios"""

39
    stragglers: list[str] = field(default_factory=lambda: ["SGLANG:EngineCore"])
40
41


42
sglang_dir = os.environ.get("SGLANG_DIR") or os.path.join(
43
    WORKSPACE_DIR, "examples/backends/sglang"
44
)
45

46
47
# SGLang test configurations
# NOTE: pytest.mark.gpu_1 tests take ~167s (2m 47s) total to run sequentially (with models pre-cached)
48
# TODO: Now that these tests use dynamic ports and each config has a profiled_vram_gib marker,
49
# optimize the runtime by bin-packing multiple engine deployments in parallel on the same GPU.
50
# A future collector/launcher can sum profiled_vram_gib values to decide how many tests fit
51
# concurrently without exceeding available VRAM.
52
53
sglang_configs = {
    "aggregated": SGLangConfig(
54
55
        # Uses backend agg.sh (with metrics enabled) for testing standard
        # aggregated deployment with metrics collection
56
        name="aggregated",
57
58
        directory=sglang_dir,
        script_name="agg.sh",
59
60
        marks=[
            pytest.mark.gpu_1,
61
62
63
64
65
66
67
            pytest.mark.profiled_vram_gib(
                3.7
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                96
            ),  # KV cache cap (2x safety over min=48)
            pytest.mark.timeout(195),  # profiled 33s on RTX 6000 Ada
68
69
            pytest.mark.pre_merge,
        ],
70
        model="Qwen/Qwen3-0.6B",
71
        env={},
72
        frontend_port=DefaultPort.FRONTEND.value,
73
74
75
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
76
77
            responses_payload_default(),
            responses_stream_payload_default(),
78
            metric_payload_default(min_num_requests=6, backend="sglang"),
79
        ],
80
81
    ),
    "disaggregated": SGLangConfig(
82
83
84
        name="disaggregated",
        directory=sglang_dir,
        script_name="disagg.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
85
86
        marks=[
            pytest.mark.gpu_2,
87
            pytest.mark.pre_merge,
88
        ],  # TODO(gpu_2): profile max_vram, timeout, add markers (separate PR)
89
90
        model="Qwen/Qwen3-0.6B",
        env={},
91
        frontend_port=DefaultPort.FRONTEND.value,
92
93
94
95
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
96
    ),
97
98
    "disaggregated_same_gpu": SGLangConfig(
        # Uses disagg_same_gpu.sh for single-GPU disaggregated testing
99
100
        # Validates metrics from both prefill (DefaultPort.SYSTEM1) and decode
        # (DefaultPort.SYSTEM2) workers
101
102
103
        name="disaggregated_same_gpu",
        directory=sglang_dir,
        script_name="disagg_same_gpu.sh",
104
105
106
107
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
            pytest.mark.skip(reason="unstable"),
108
            # TODO: profile to get max_vram and timeout (currently skipped)
109
        ],
110
        model="Qwen/Qwen3-0.6B",
111
        delayed_start=30,
112
        env={},
113
        frontend_port=DefaultPort.FRONTEND.value,
114
115
116
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
117
118
119
120
121
122
123
124
125
126
127
128
129
130
            # Validate dynamo_component_* and sglang:* metrics from prefill worker
            # (DefaultPort.SYSTEM1)
            metric_payload_default(
                min_num_requests=6,
                backend="sglang",
                port=DefaultPort.SYSTEM1.value,
            ),
            # Validate dynamo_component_* and sglang:* metrics from decode worker
            # (DefaultPort.SYSTEM2)
            metric_payload_default(
                min_num_requests=6,
                backend="sglang",
                port=DefaultPort.SYSTEM2.value,
            ),
131
132
        ],
    ),
133
    "kv_events": SGLangConfig(
134
135
136
        name="kv_events",
        directory=sglang_dir,
        script_name="agg_router.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
137
138
        marks=[
            pytest.mark.gpu_2,
139
            pytest.mark.pre_merge,
140
        ],  # TODO(gpu_2): profile max_vram, timeout, add markers (separate PR)
141
142
        model="Qwen/Qwen3-0.6B",
        env={
143
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_kv_router::scheduling::selector=info",
144
        },
145
        frontend_port=DefaultPort.FRONTEND.value,
146
147
148
        request_payloads=[
            chat_payload_default(
                expected_log=[
149
                    r"ZMQ listener .* received batch with \d+ events \(engine_seq=\d+(?:, [^)]*)?\)",
150
                    r"Event processor for worker_id \d+ processing event: Stored\(",
151
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
152
153
154
                ]
            )
        ],
155
    ),
156
157
158
159
160
    "template_verification": SGLangConfig(
        # Tests custom jinja template preprocessing by verifying the template
        # marker 'CUSTOM_TEMPLATE_ACTIVE|' is applied to user messages.
        # The backend (launch/template_verifier.*) checks for this marker
        # and returns "Successfully Applied Chat Template" if found.
161
162
        # Uses SERVE_TEST_DIR (not sglang_dir) because template_verifier.sh/.py
        # are test-specific mock scripts in tests/serve/launch/
163
        name="template_verification",
164
        directory=SERVE_TEST_DIR,  # special directory for test-specific scripts
165
        script_name="template_verifier.sh",
166
167
        marks=[
            pytest.mark.gpu_1,
168
169
            pytest.mark.profiled_vram_gib(0.0),  # no GPU model load
            pytest.mark.timeout(120),  # profiled 12s on RTX 6000 Ada
170
171
172
            pytest.mark.pre_merge,
            pytest.mark.nightly,
        ],
173
174
        model="Qwen/Qwen3-0.6B",
        env={},
175
        frontend_port=DefaultPort.FRONTEND.value,
176
177
178
179
180
181
        request_payloads=[
            chat_payload_default(
                expected_response=["Successfully Applied Chat Template"]
            )
        ],
    ),
182
183
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements.
    # NOTE: multimodal_epd.sh uses explicit --mem-fraction-static via DYN_ENCODE_GPU_MEM
184
185
    # / DYN_WORKER_GPU_MEM env vars. The profiler override distributes proportionally
    # but workers combined consistently use ~23.6 GiB regardless of fraction overrides.
186
    "multimodal_e_pd_qwen": SGLangConfig(
187
        # E/P/D architecture: Encode, Prefill, Decode workers all on GPU 0
188
        name="multimodal_e_pd_qwen",
189
        directory=sglang_dir,
190
        script_name="multimodal_epd.sh",
191
192
        marks=[
            pytest.mark.gpu_1,
193
194
195
            # No profiled_vram_gib: uses hard-coded --mem-fraction-static via
            # DYN_ENCODE_GPU_MEM / DYN_WORKER_GPU_MEM, so VRAM scales with GPU size.
            pytest.mark.timeout(210),  # profiled 35s on RTX 6000 Ada
196
197
            pytest.mark.pre_merge,
        ],
198
199
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
200
        timeout=360,
201
202
203
204
        env={
            "DYN_ENCODE_GPU_MEM": "0.1",
            "DYN_WORKER_GPU_MEM": "0.4",
        },
205
        frontend_port=DefaultPort.FRONTEND.value,
206
207
208
209
210
211
212
213
214
215
216
217
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
218
219
220
                # NOTE: The response text may mention 'bus', 'train', 'streetcar', etc.
                # so we need something consistently found in the response, or a different
                # approach to validation for this test to be stable.
221
                expected_response=["image"],
222
                temperature=0.0,
223
                max_tokens=100,
224
225
226
            )
        ],
    ),
227
228
229
230
231
232
233
    "multimodal_disagg_qwen": SGLangConfig(
        # E/P/D architecture: Encode, Prefill, Decode workers all on GPU 0
        name="multimodal_disagg_qwen",
        directory=sglang_dir,
        script_name="multimodal_disagg.sh",
        marks=[
            pytest.mark.gpu_1,
234
235
236
237
238
            pytest.mark.profiled_vram_gib(16.1),  # actual profiled peak
            pytest.mark.requested_sglang_kv_tokens(
                1024
            ),  # KV cache cap (2x safety over min=512)
            pytest.mark.timeout(222),  # profiled 37s on RTX 6000 Ada
239
240
241
242
243
            pytest.mark.pre_merge,
        ],
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
        timeout=360,
244
        env={},
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["image"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
264
265
266
267
268
269
270
271
    "multimodal_agg_qwen": SGLangConfig(
        # Tests single-process aggregated multimodal inference using DecodeWorkerHandler
        # with in-process vision encoding (no separate encode worker)
        name="multimodal_agg_qwen",
        directory=sglang_dir,
        script_name="agg.sh",
        marks=[
            pytest.mark.gpu_1,
272
273
274
275
276
277
278
            pytest.mark.profiled_vram_gib(
                19.1
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                768
            ),  # KV cache cap (2x safety over min=384)
            pytest.mark.timeout(182),  # profiled 30s on RTX 6000 Ada
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
            pytest.mark.pre_merge,
            pytest.mark.nightly,
        ],
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=[
            "--model-path",
            "Qwen/Qwen2.5-VL-7B-Instruct",
            "--chat-template",
            "qwen2-vl",
        ],
        delayed_start=0,
        timeout=360,
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["image"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
310
311
312
313
    "embedding_agg": SGLangConfig(
        name="embedding_agg",
        directory=sglang_dir,
        script_name="agg_embed.sh",
314
315
        marks=[
            pytest.mark.gpu_1,
316
317
318
319
320
321
322
            pytest.mark.profiled_vram_gib(
                9.8
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                128
            ),  # KV cache cap (2x safety over min=64)
            pytest.mark.timeout(147),  # profiled 24s on RTX 6000 Ada
323
324
325
            pytest.mark.pre_merge,
            pytest.mark.nightly,
        ],
326
327
        model="Qwen/Qwen3-Embedding-4B",
        delayed_start=0,
328
        frontend_port=DefaultPort.FRONTEND.value,
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
        request_payloads=[
            # Test default payload with multiple inputs
            embedding_payload_default(
                repeat_count=2,
                expected_response=["Generated 2 embeddings with dimension"],
            ),
            # Test single string input
            embedding_payload(
                input_text="Hello, world!",
                repeat_count=1,
                expected_response=["Generated 1 embeddings with dimension"],
            ),
            # Test multiple string inputs
            embedding_payload(
                input_text=[
                    "The quick brown fox jumps over the lazy dog.",
                    "Machine learning is transforming technology.",
                    "Natural language processing enables computers to understand text.",
                ],
                repeat_count=1,
                expected_response=["Generated 3 embeddings with dimension"],
            ),
        ],
    ),
353
354
355
356
    "completions_only": SGLangConfig(
        name="completions_only",
        directory=sglang_dir,
        script_name="agg.sh",
357
358
        marks=[
            pytest.mark.gpu_1,
359
360
361
362
363
364
365
            pytest.mark.profiled_vram_gib(
                14.7
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                64
            ),  # KV cache cap (2x safety over min=32)
            pytest.mark.timeout(341),  # profiled 57s on RTX 6000 Ada
366
            pytest.mark.post_merge,
367
        ],
368
369
370
371
372
373
374
375
376
377
378
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model-path",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
379
380
381
382
383
384
385
386
    "anthropic_messages": SGLangConfig(
        name="anthropic_messages",
        directory=sglang_dir,
        script_name="agg.sh",
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.post_merge,
            pytest.mark.timeout(240),
Dmitry Tokarev's avatar
Dmitry Tokarev committed
387
            pytest.mark.skip(reason="DYN-2261"),
388
            # TODO: profile once DYN-2261 is fixed (uses agg.sh, profiler works)
389
390
391
392
393
394
395
396
397
        ],
        model="Qwen/Qwen3-0.6B",
        env={"DYN_ENABLE_ANTHROPIC_API": "1"},
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            anthropic_messages_payload_default(),
            anthropic_messages_stream_payload_default(),
        ],
    ),
398
399
400
}


Alec's avatar
Alec committed
401
@pytest.fixture(params=params_with_model_mark(sglang_configs))
402
403
404
405
406
407
408
def sglang_config_test(request):
    """Fixture that provides different SGLang test configurations"""
    return sglang_configs[request.param]


@pytest.mark.e2e
@pytest.mark.sglang
409
410
411
# Use 2 system ports because some `sglang_configs` validate metrics on multiple ports.
# This test iterates over all configs via `sglang_config_test`.
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
Alec's avatar
Alec committed
412
def test_sglang_deployment(
413
414
415
416
    sglang_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
417
    num_system_ports,
418
    predownload_models,
Alec's avatar
Alec committed
419
):
420
    """Test SGLang deployment scenarios using common helpers"""
421
422
423
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
424
425
426
427
    config = dataclasses.replace(
        sglang_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
428
429


430
431
@pytest.mark.e2e
@pytest.mark.sglang
432
@pytest.mark.gpu_2
433
@pytest.mark.nightly
434
435
436
@pytest.mark.skip(
    reason="Requires 4 GPUs - enable when hardware is consistently available"
)
437
438
439
def test_sglang_disagg_dp_attention(
    request, runtime_services_dynamic_ports, dynamo_dynamic_ports, predownload_models
):
440
441
    """Test sglang disaggregated with DP attention (requires 4 GPUs)"""

442
    # Kept for reference; this test uses a different launch path and is skipped