test_sglang.py 17.6 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import dataclasses
5
6
import logging
import os
7
from dataclasses import dataclass, field
8
9
10

import pytest

Alec's avatar
Alec committed
11
12
from tests.serve.common import (
    SERVE_TEST_DIR,
13
    WORKSPACE_DIR,
Alec's avatar
Alec committed
14
15
16
    params_with_model_mark,
    run_serve_deployment,
)
17
from tests.utils.constants import DefaultPort
18
from tests.utils.engine_process import EngineConfig
19
from tests.utils.payload_builder import (
20
21
    anthropic_messages_payload_default,
    anthropic_messages_stream_payload_default,
22
23
24
    chat_payload,
    chat_payload_default,
    completion_payload_default,
25
26
    embedding_payload,
    embedding_payload_default,
27
    metric_payload_default,
28
29
    responses_payload_default,
    responses_stream_payload_default,
30
)
31
32
33
34
35

logger = logging.getLogger(__name__)


@dataclass
36
class SGLangConfig(EngineConfig):
37
38
    """Configuration for SGLang test scenarios"""

39
    stragglers: list[str] = field(default_factory=lambda: ["SGLANG:EngineCore"])
40
41


42
sglang_dir = os.environ.get("SGLANG_DIR") or os.path.join(
43
    WORKSPACE_DIR, "examples/backends/sglang"
44
)
45
46
47
REMOTE_VIDEO_TEST_URI = (
    "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4"
)
48

49
50
# SGLang test configurations
# NOTE: pytest.mark.gpu_1 tests take ~167s (2m 47s) total to run sequentially (with models pre-cached)
51
# TODO: Now that these tests use dynamic ports and each config has a profiled_vram_gib marker,
52
# optimize the runtime by bin-packing multiple engine deployments in parallel on the same GPU.
53
# A future collector/launcher can sum profiled_vram_gib values to decide how many tests fit
54
# concurrently without exceeding available VRAM.
55
56
sglang_configs = {
    "aggregated": SGLangConfig(
57
58
        # Uses backend agg.sh (with metrics enabled) for testing standard
        # aggregated deployment with metrics collection
59
        name="aggregated",
60
61
        directory=sglang_dir,
        script_name="agg.sh",
62
63
        marks=[
            pytest.mark.gpu_1,
64
65
66
67
68
69
70
            pytest.mark.profiled_vram_gib(
                3.7
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                96
            ),  # KV cache cap (2x safety over min=48)
            pytest.mark.timeout(195),  # profiled 33s on RTX 6000 Ada
71
72
            pytest.mark.pre_merge,
        ],
73
        model="Qwen/Qwen3-0.6B",
74
        env={},
75
        frontend_port=DefaultPort.FRONTEND.value,
76
77
78
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
79
80
            responses_payload_default(),
            responses_stream_payload_default(),
81
            metric_payload_default(min_num_requests=6, backend="sglang"),
82
        ],
83
84
    ),
    "disaggregated": SGLangConfig(
85
86
87
        name="disaggregated",
        directory=sglang_dir,
        script_name="disagg.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
88
89
        marks=[
            pytest.mark.gpu_2,
90
            pytest.mark.pre_merge,
91
        ],  # TODO(gpu_2): profile max_vram, timeout, add markers (separate PR)
92
93
        model="Qwen/Qwen3-0.6B",
        env={},
94
        frontend_port=DefaultPort.FRONTEND.value,
95
96
97
98
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
99
    ),
100
101
    "disaggregated_same_gpu": SGLangConfig(
        # Uses disagg_same_gpu.sh for single-GPU disaggregated testing
102
103
        # Validates metrics from both prefill (DefaultPort.SYSTEM1) and decode
        # (DefaultPort.SYSTEM2) workers
104
105
106
        name="disaggregated_same_gpu",
        directory=sglang_dir,
        script_name="disagg_same_gpu.sh",
107
108
109
110
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
            pytest.mark.skip(reason="unstable"),
111
            # TODO: profile to get max_vram and timeout (currently skipped)
112
        ],
113
        model="Qwen/Qwen3-0.6B",
114
        delayed_start=30,
115
        env={},
116
        frontend_port=DefaultPort.FRONTEND.value,
117
118
119
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
120
121
122
123
124
125
126
127
128
129
130
131
132
133
            # Validate dynamo_component_* and sglang:* metrics from prefill worker
            # (DefaultPort.SYSTEM1)
            metric_payload_default(
                min_num_requests=6,
                backend="sglang",
                port=DefaultPort.SYSTEM1.value,
            ),
            # Validate dynamo_component_* and sglang:* metrics from decode worker
            # (DefaultPort.SYSTEM2)
            metric_payload_default(
                min_num_requests=6,
                backend="sglang",
                port=DefaultPort.SYSTEM2.value,
            ),
134
135
        ],
    ),
136
    "kv_events": SGLangConfig(
137
138
139
        name="kv_events",
        directory=sglang_dir,
        script_name="agg_router.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
140
141
        marks=[
            pytest.mark.gpu_2,
142
            pytest.mark.pre_merge,
143
        ],  # TODO(gpu_2): profile max_vram, timeout, add markers (separate PR)
144
145
        model="Qwen/Qwen3-0.6B",
        env={
146
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_kv_router::scheduling::selector=info",
147
        },
148
        frontend_port=DefaultPort.FRONTEND.value,
149
150
151
        request_payloads=[
            chat_payload_default(
                expected_log=[
152
                    r"ZMQ listener .* received batch with \d+ events \(engine_seq=\d+(?:, [^)]*)?\)",
153
                    r"Event processor for worker_id \d+ processing event: Stored\(",
154
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
155
156
157
                ]
            )
        ],
158
    ),
159
160
161
162
163
    "template_verification": SGLangConfig(
        # Tests custom jinja template preprocessing by verifying the template
        # marker 'CUSTOM_TEMPLATE_ACTIVE|' is applied to user messages.
        # The backend (launch/template_verifier.*) checks for this marker
        # and returns "Successfully Applied Chat Template" if found.
164
165
        # Uses SERVE_TEST_DIR (not sglang_dir) because template_verifier.sh/.py
        # are test-specific mock scripts in tests/serve/launch/
166
        name="template_verification",
167
        directory=SERVE_TEST_DIR,  # special directory for test-specific scripts
168
        script_name="template_verifier.sh",
169
170
        marks=[
            pytest.mark.gpu_1,
171
172
            pytest.mark.profiled_vram_gib(0.0),  # no GPU model load
            pytest.mark.timeout(120),  # profiled 12s on RTX 6000 Ada
173
174
175
            pytest.mark.pre_merge,
            pytest.mark.nightly,
        ],
176
177
        model="Qwen/Qwen3-0.6B",
        env={},
178
        frontend_port=DefaultPort.FRONTEND.value,
179
180
181
182
183
184
        request_payloads=[
            chat_payload_default(
                expected_response=["Successfully Applied Chat Template"]
            )
        ],
    ),
185
186
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements.
    # NOTE: multimodal_epd.sh uses explicit --mem-fraction-static via DYN_ENCODE_GPU_MEM
187
188
    # / DYN_WORKER_GPU_MEM env vars. The profiler override distributes proportionally
    # but workers combined consistently use ~23.6 GiB regardless of fraction overrides.
189
    "multimodal_e_pd_qwen": SGLangConfig(
190
        # E/P/D architecture: Encode, Prefill, Decode workers all on GPU 0
191
        name="multimodal_e_pd_qwen",
192
        directory=sglang_dir,
193
        script_name="multimodal_epd.sh",
194
195
        marks=[
            pytest.mark.gpu_1,
196
197
198
            # No profiled_vram_gib: uses hard-coded --mem-fraction-static via
            # DYN_ENCODE_GPU_MEM / DYN_WORKER_GPU_MEM, so VRAM scales with GPU size.
            pytest.mark.timeout(210),  # profiled 35s on RTX 6000 Ada
199
200
            pytest.mark.pre_merge,
        ],
201
202
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
203
        timeout=360,
204
205
206
207
        env={
            "DYN_ENCODE_GPU_MEM": "0.1",
            "DYN_WORKER_GPU_MEM": "0.4",
        },
208
        frontend_port=DefaultPort.FRONTEND.value,
209
210
211
212
213
214
215
216
217
218
219
220
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
221
222
223
                # NOTE: The response text may mention 'bus', 'train', 'streetcar', etc.
                # so we need something consistently found in the response, or a different
                # approach to validation for this test to be stable.
224
                expected_response=["image"],
225
                temperature=0.0,
226
                max_tokens=100,
227
228
229
            )
        ],
    ),
230
231
232
233
234
235
236
    "multimodal_disagg_qwen": SGLangConfig(
        # E/P/D architecture: Encode, Prefill, Decode workers all on GPU 0
        name="multimodal_disagg_qwen",
        directory=sglang_dir,
        script_name="multimodal_disagg.sh",
        marks=[
            pytest.mark.gpu_1,
237
238
239
240
241
            pytest.mark.profiled_vram_gib(16.1),  # actual profiled peak
            pytest.mark.requested_sglang_kv_tokens(
                1024
            ),  # KV cache cap (2x safety over min=512)
            pytest.mark.timeout(222),  # profiled 37s on RTX 6000 Ada
242
243
244
245
246
            pytest.mark.pre_merge,
        ],
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
        timeout=360,
247
        env={},
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["image"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
267
268
269
270
271
272
273
    "multimodal_agg_qwen": SGLangConfig(
        # Tests single-process aggregated multimodal inference using DecodeWorkerHandler
        # with in-process vision encoding (no separate encode worker)
        name="multimodal_agg_qwen",
        directory=sglang_dir,
        script_name="agg.sh",
        marks=[
274
275
276
            pytest.mark.skip(
                reason="Nightly CI failure: https://linear.app/nvidia/issue/DYN-2602"
            ),
277
            pytest.mark.gpu_1,
278
279
280
281
282
283
284
            pytest.mark.profiled_vram_gib(
                19.1
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                768
            ),  # KV cache cap (2x safety over min=384)
            pytest.mark.timeout(182),  # profiled 30s on RTX 6000 Ada
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
            pytest.mark.pre_merge,
            pytest.mark.nightly,
        ],
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=[
            "--model-path",
            "Qwen/Qwen2.5-VL-7B-Instruct",
            "--chat-template",
            "qwen2-vl",
        ],
        delayed_start=0,
        timeout=360,
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["image"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
    "video_agg_qwen": SGLangConfig(
        # Tests aggregated video inference using DecodeWorkerHandler
        # with in-process vision encoding (no separate encode worker).
        # Reuses agg_vision.sh because image and video share the same aggregated
        # multimodal SGLang request path.
        name="video_agg_qwen",
        directory=sglang_dir,
        script_name="agg_vision.sh",
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.profiled_vram_gib(13.3),  # same as multimodal_e_pd_qwen
            pytest.mark.timeout(360),
            pytest.mark.pre_merge,
        ],
        model="Qwen/Qwen2-VL-7B-Instruct",
        script_args=[
            "--model-path",
            "Qwen/Qwen2-VL-7B-Instruct",
            "--mem-fraction-static",
            "0.8",
        ],
        timeout=360,
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {"url": REMOTE_VIDEO_TEST_URI},
                    },
                ],
                repeat_count=1,
                expected_response=["guitar", "tablet", "draw"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
355
356
357
358
    "embedding_agg": SGLangConfig(
        name="embedding_agg",
        directory=sglang_dir,
        script_name="agg_embed.sh",
359
360
        marks=[
            pytest.mark.gpu_1,
361
362
363
364
365
366
367
            pytest.mark.profiled_vram_gib(
                9.8
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                128
            ),  # KV cache cap (2x safety over min=64)
            pytest.mark.timeout(147),  # profiled 24s on RTX 6000 Ada
368
369
370
            pytest.mark.pre_merge,
            pytest.mark.nightly,
        ],
371
372
        model="Qwen/Qwen3-Embedding-4B",
        delayed_start=0,
373
        frontend_port=DefaultPort.FRONTEND.value,
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
        request_payloads=[
            # Test default payload with multiple inputs
            embedding_payload_default(
                repeat_count=2,
                expected_response=["Generated 2 embeddings with dimension"],
            ),
            # Test single string input
            embedding_payload(
                input_text="Hello, world!",
                repeat_count=1,
                expected_response=["Generated 1 embeddings with dimension"],
            ),
            # Test multiple string inputs
            embedding_payload(
                input_text=[
                    "The quick brown fox jumps over the lazy dog.",
                    "Machine learning is transforming technology.",
                    "Natural language processing enables computers to understand text.",
                ],
                repeat_count=1,
                expected_response=["Generated 3 embeddings with dimension"],
            ),
        ],
    ),
398
399
400
401
    "completions_only": SGLangConfig(
        name="completions_only",
        directory=sglang_dir,
        script_name="agg.sh",
402
403
        marks=[
            pytest.mark.gpu_1,
404
405
406
407
408
409
410
            pytest.mark.profiled_vram_gib(
                14.7
            ),  # actual peak at recommended token count
            pytest.mark.requested_sglang_kv_tokens(
                64
            ),  # KV cache cap (2x safety over min=32)
            pytest.mark.timeout(341),  # profiled 57s on RTX 6000 Ada
411
            pytest.mark.post_merge,
412
        ],
413
414
415
416
417
418
419
420
421
422
423
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model-path",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
424
425
426
427
428
429
430
431
    "anthropic_messages": SGLangConfig(
        name="anthropic_messages",
        directory=sglang_dir,
        script_name="agg.sh",
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.post_merge,
            pytest.mark.timeout(240),
Dmitry Tokarev's avatar
Dmitry Tokarev committed
432
            pytest.mark.skip(reason="DYN-2261"),
433
            # TODO: profile once DYN-2261 is fixed (uses agg.sh, profiler works)
434
435
436
437
438
439
440
441
442
        ],
        model="Qwen/Qwen3-0.6B",
        env={"DYN_ENABLE_ANTHROPIC_API": "1"},
        frontend_port=DefaultPort.FRONTEND.value,
        request_payloads=[
            anthropic_messages_payload_default(),
            anthropic_messages_stream_payload_default(),
        ],
    ),
443
444
445
}


Alec's avatar
Alec committed
446
@pytest.fixture(params=params_with_model_mark(sglang_configs))
447
448
449
450
451
452
453
def sglang_config_test(request):
    """Fixture that provides different SGLang test configurations"""
    return sglang_configs[request.param]


@pytest.mark.e2e
@pytest.mark.sglang
454
455
456
# Use 2 system ports because some `sglang_configs` validate metrics on multiple ports.
# This test iterates over all configs via `sglang_config_test`.
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
Alec's avatar
Alec committed
457
def test_sglang_deployment(
458
459
460
461
    sglang_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
462
    num_system_ports,
463
    predownload_models,
Alec's avatar
Alec committed
464
):
465
    """Test SGLang deployment scenarios using common helpers"""
466
467
468
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
469
470
471
472
    config = dataclasses.replace(
        sglang_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
473
474


475
476
@pytest.mark.e2e
@pytest.mark.sglang
477
@pytest.mark.gpu_2
478
@pytest.mark.nightly
479
480
481
@pytest.mark.skip(
    reason="Requires 4 GPUs - enable when hardware is consistently available"
)
482
483
484
def test_sglang_disagg_dp_attention(
    request, runtime_services_dynamic_ports, dynamo_dynamic_ports, predownload_models
):
485
486
    """Test sglang disaggregated with DP attention (requires 4 GPUs)"""

487
    # Kept for reference; this test uses a different launch path and is skipped