test_vllm.py 41 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import base64
5
import dataclasses
6
7
import logging
import os
8
import random
9
from dataclasses import dataclass, field
10
from pathlib import Path
11
from typing import Optional
12
13
14

import pytest

15
16
17
18
19
from tests.serve.common import (
    WORKSPACE_DIR,
    params_with_model_mark,
    run_serve_deployment,
)
20
from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes
21
from tests.serve.lora_utils import MinioLoraConfig
22
from tests.utils.constants import DefaultPort
23
24
from tests.utils.engine_process import EngineConfig
from tests.utils.payload_builder import (
25
    cached_tokens_chat_payload,
26
27
    chat_payload,
    chat_payload_default,
28
    chat_payload_with_logprobs,
29
    completion_payload_default,
30
    completion_payload_with_logprobs,
31
    metric_payload_default,
32
)
33
from tests.utils.payloads import LoraTestChatPayload, ToolCallingChatPayload
34
35
36
37

logger = logging.getLogger(__name__)


38
39
40
41
42
43
def _is_cuda13() -> bool:
    v = os.environ.get("CUDA_VERSION", "")
    # handles "13", "13.0", "13.0.1", etc.
    return v.startswith("13")


44
@dataclass
45
class VLLMConfig(EngineConfig):
46
47
    """Configuration for vLLM test scenarios"""

48
    stragglers: list[str] = field(default_factory=lambda: ["VLLM:EngineCore"])
49
50


51
vllm_dir = os.environ.get("VLLM_DIR") or os.path.join(
52
    WORKSPACE_DIR, "examples/backends/vllm"
53
)
54
55
56
57
LOCAL_VIDEO_TEST_PATH = Path(
    WORKSPACE_DIR, "lib/llm/tests/data/media/240p_10.mp4"
).resolve()
LOCAL_VIDEO_TEST_URI = LOCAL_VIDEO_TEST_PATH.as_uri()
58

59

60
# vLLM test configurations
61
# NOTE: pytest.mark.gpu_1 tests take ~5.5 minutes total to run sequentially (with models pre-cached)
62
# TODO: Now that these tests use dynamic ports and each config has VRAM markers,
63
# optimize the runtime by bin-packing multiple engine deployments in parallel on the same GPU.
64
# A future collector/launcher can sum profiled_vram_gib values to decide how many tests fit
65
# concurrently without exceeding available VRAM.
66
67
68
vllm_configs = {
    "aggregated": VLLMConfig(
        name="aggregated",
69
        directory=vllm_dir,
70
        script_name="agg.sh",
71
72
        marks=[
            pytest.mark.gpu_1,
73
74
75
76
77
78
79
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
            pytest.mark.timeout(
                360
            ),  # ~8.5x observed 42.2s; bumped for GPU-parallel headroom
80
81
            pytest.mark.pre_merge,
        ],
82
        model="Qwen/Qwen3-0.6B",
83
84
85
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
86
87
88
89
90
91
92
93
94
95
96
            chat_payload(
                "Can you write me a song?",
                repeat_count=1,
                expected_response=["song"],
                temperature=0.0,
                max_tokens=32,
                extra_body={
                    "stop": ["song"],
                    "include_stop_str_in_output": True,
                },
            ),
97
            metric_payload_default(min_num_requests=6, backend="vllm"),
98
        ],
99
    ),
100
101
102
103
    "aggregated_logprobs": VLLMConfig(
        name="aggregated_logprobs",
        directory=vllm_dir,
        script_name="agg.sh",
104
105
        marks=[
            pytest.mark.gpu_1,
106
107
108
109
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
110
            pytest.mark.timeout(120),  # ~5x observed 24.3s; CI machines are slower
111
112
            pytest.mark.post_merge,
        ],
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                top_logprobs=3,
            ),
            completion_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                logprobs=5,
            ),
        ],
    ),
131
132
133
134
    "aggregated_lmcache": VLLMConfig(
        name="aggregated_lmcache",
        directory=vllm_dir,
        script_name="agg_lmcache.sh",
135
        marks=[
136
            pytest.mark.lmcache,
137
            pytest.mark.gpu_1,
138
139
140
141
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
142
            pytest.mark.timeout(360),  # ~7x observed 49.0s; old value before profiling
143
            pytest.mark.pre_merge,
144
            pytest.mark.skipif(
145
146
147
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
            ),
148
        ],
149
150
151
152
153
154
155
156
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
157
158
159
160
    "aggregated_lmcache_multiproc": VLLMConfig(
        name="aggregated_lmcache_multiproc",
        directory=vllm_dir,
        script_name="agg_lmcache_multiproc.sh",
161
        marks=[
162
            pytest.mark.lmcache,
163
            pytest.mark.gpu_1,
164
165
166
167
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
168
            pytest.mark.timeout(360),  # ~7x observed 49.3s; old value before profiling
169
            pytest.mark.pre_merge,
170
            pytest.mark.skipif(
171
172
173
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
            ),
174
        ],
175
176
177
178
179
180
181
182
183
184
185
        model="Qwen/Qwen3-0.6B",
        env={
            "PROMETHEUS_MULTIPROC_DIR": f"/tmp/prometheus_multiproc_test_{os.getpid()}_{random.randint(0, 10000)}"
        },
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
186
187
188
189
    "agg-request-plane-tcp": VLLMConfig(
        name="agg-request-plane-tcp",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
190
191
        marks=[
            pytest.mark.gpu_1,
192
193
194
195
196
197
198
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
            pytest.mark.timeout(
                360
            ),  # ~8x observed 43.0s; bumped for GPU-parallel headroom
199
200
            pytest.mark.pre_merge,
        ],
201
202
203
204
205
206
207
208
209
210
211
        model="Qwen/Qwen3-0.6B",
        script_args=["--tcp"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
    "agg-request-plane-http": VLLMConfig(
        name="agg-request-plane-http",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
212
213
        marks=[
            pytest.mark.gpu_1,
214
215
216
217
218
219
220
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
            pytest.mark.timeout(
                360
            ),  # ~8.5x observed 42.3s; bumped for GPU-parallel headroom
221
222
            pytest.mark.pre_merge,
        ],
223
224
225
226
227
228
229
        model="Qwen/Qwen3-0.6B",
        script_args=["--http"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
230
231
    "agg-router": VLLMConfig(
        name="agg-router",
232
        directory=vllm_dir,
233
        script_name="agg_router.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
234
235
        marks=[
            pytest.mark.gpu_2,
236
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
237
            pytest.mark.skip(reason="DYN-2263"),
238
        ],  # TODO: profile to get max_vram and timeout
239
        model="Qwen/Qwen3-0.6B",
240
241
242
        request_payloads=[
            chat_payload_default(
                expected_log=[
243
                    r"ZMQ listener .* received batch with \d+ events \(seq=\d+(?:, [^)]*)?\)",
244
                    r"Event processor for worker_id \d+ processing event: Stored\(",
245
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
246
247
248
249
                ]
            )
        ],
        env={
250
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_kv_router::scheduling::selector=info",
251
        },
252
    ),
253
254
255
256
    "agg-router-approx": VLLMConfig(
        name="agg-router-approx",
        directory=vllm_dir,
        script_name="agg_router_approx.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
257
258
        marks=[
            pytest.mark.gpu_2,
259
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
260
            pytest.mark.skip(reason="DYN-2264"),
261
        ],  # TODO: profile to get max_vram and timeout
262
263
264
265
266
267
268
269
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            # Test approximate KV routing (--no-kv-events mode)
            # Repeated requests should show cache-aware routing in logs
            chat_payload_default(
                repeat_count=3,
                expected_log=[
                    # Verify scheduler is selecting workers with cache awareness
270
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
271
272
273
274
275
276
277
278
279
280
281
282
283
284
                    # After first request, should see cached blocks being tracked
                    r"with \d+ cached blocks",
                ],
            ),
            # Also test with cached tokens payload to verify usage field
            cached_tokens_chat_payload(
                repeat_count=3,
                expected_log=[
                    # Verify routing decision shows cache hits
                    r"with \d+ cached blocks",
                ],
            ),
        ],
        env={
285
            "DYN_LOG": "dynamo_kv_router::scheduling::selector=info",
286
287
        },
    ),
288
289
    "disaggregated": VLLMConfig(
        name="disaggregated",
290
        directory=vllm_dir,
291
        script_name="disagg.sh",
292
293
294
295
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.pre_merge,
        ],  # TODO: profile to get max_vram and timeout
296
        model="Qwen/Qwen3-0.6B",
297
298
299
300
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
301
    ),
302
303
    "deepep": VLLMConfig(
        name="deepep",
304
        directory=vllm_dir,
305
        script_name="dsr1_dep.sh",
306
307
308
309
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.vllm,
            pytest.mark.h100,
310
            pytest.mark.nightly,
311
            # TODO: profile to get max_vram and timeout
312
        ],
313
        model="deepseek-ai/DeepSeek-V2-Lite",
314
        script_args=[
315
316
317
318
319
320
321
322
323
            "--model",
            "deepseek-ai/DeepSeek-V2-Lite",
            "--num-nodes",
            "1",
            "--node-rank",
            "0",
            "--gpus-per-node",
            "2",
        ],
324
        timeout=700,
325
        request_payloads=[
326
327
            chat_payload_default(),
            completion_payload_default(),
328
        ],
329
    ),
330
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements
331
332
    # NOTE: disagg_multimodal_e_pd.sh uses explicit --gpu-memory-utilization via
    # DYN_ENCODE_GPU_MEM / DYN_PD_GPU_MEM env vars in single-GPU mode.
333
    # PD worker honors build_vllm_gpu_mem_args for parallel execution.
334
335
    "multimodal_e_pd_qwen": VLLMConfig(
        name="multimodal_e_pd_qwen",
336
337
        directory=vllm_dir,
        script_name="disagg_multimodal_e_pd.sh",
338
339
        marks=[
            pytest.mark.gpu_1,
340
341
            # No profiled_vram_gib / requested_vllm_kv_cache_bytes: single-GPU mode
            # uses hardcoded fractions (encode=0.1, PD=0.7) that scale with GPU size.
342
            pytest.mark.timeout(340),  # ~5x observed 68.4s; 2B model loads slower on CI
343
344
            pytest.mark.pre_merge,
        ],
345
346
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                # With proper prompt templating, the model actually only returns "green",
                # verified behavior with native vLLM.
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
368
369
370
371
    "multimodal_agg_frontend_decoding": VLLMConfig(
        name="multimodal_agg_frontend_decoding",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
372
        # post_merge because needs real NIXL not stub
373
374
        marks=[
            pytest.mark.gpu_1,
375
376
377
378
            pytest.mark.profiled_vram_gib(9.6),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_710_490_000
            ),  # KV cache cap (2x safety over min=855_244_800)
379
            pytest.mark.timeout(220),  # ~5x observed 43.7s; 2B model loads slower on CI
380
381
            pytest.mark.post_merge,
        ],
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
        model="Qwen/Qwen2-VL-2B-Instruct",
        # Pass --frontend-decoding to enable Rust frontend image decoding + NIXL RDMA transfer
        script_args=[
            "--model",
            "Qwen/Qwen2-VL-2B-Instruct",
            "--frontend-decoding",
        ],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
408
409
410
411
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements.
    # NOTE: disagg_multimodal_epd.sh uses --kv-cache-memory-bytes=512MB for P/D
    # workers. Per vLLM CacheConfig, kv_cache_memory_bytes (when not-None) ignores
    # gpu_memory_utilization (ref: https://docs.vllm.ai/en/stable/api/vllm/config/cache/),
412
    # so KV cache overrides have no effect. Regardless of GPU_MEM
413
414
    # fractions (0.1/0.4/0.4), the 3 workers combined consistently use ~17.6 GiB
    # total on this GPU.
415
416
    # NOTE: disagg_multimodal_epd.sh uses explicit --gpu-memory-utilization via
    # DYN_ENCODE_GPU_MEM / DYN_PREFILL_GPU_MEM / DYN_DECODE_GPU_MEM env vars.
417
    # P/D workers honor build_vllm_gpu_mem_args for parallel execution.
418
419
    "multimodal_disagg_qwen": VLLMConfig(
        name="multimodal_disagg_qwen",
420
        directory=vllm_dir,
421
        script_name="disagg_multimodal_epd.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
422
423
        marks=[
            pytest.mark.gpu_1,
424
425
            # No profiled_vram_gib / requested_vllm_kv_cache_bytes: single-GPU mode
            # uses hardcoded fractions via DYN_*_GPU_MEM that scale with GPU size.
426
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
427
        ],
428
        model="Qwen/Qwen3-VL-2B-Instruct",
429
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
430
        timeout=300,
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    # P/D multimodal (no encoder): prefill loads images via PIL,
    # computes grid_thw for decode using smart_resize.
    "multimodal_p_d_qwen": VLLMConfig(
        name="multimodal_p_d_qwen",
        directory=vllm_dir,
        script_name="disagg_multimodal_p_d.sh",
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
        ],
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
        timeout=300,
463
464
465
        request_payloads=[
            chat_payload(
                [
466
467
468
469
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
470
471
                    {
                        "type": "image_url",
472
                        "image_url": {"url": MULTIMODAL_IMG_URL},
473
474
475
                    },
                ],
                repeat_count=1,
476
                expected_response=["green"],
477
                temperature=0.0,
478
                max_tokens=100,
479
480
            )
        ],
481
    ),
482
483
    "multimodal_agg_qwen": VLLMConfig(
        name="multimodal_agg_qwen",
484
485
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
486
487
        marks=[
            pytest.mark.gpu_1,
488
489
490
491
            pytest.mark.profiled_vram_gib(19.9),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                922_354_000
            ),  # KV cache cap (2x safety over min=461_176_832)
492
493
494
            pytest.mark.timeout(
                360
            ),  # ~7x observed 50.0s; 7B model loads ~48s on CI (A10G/L4)
495
496
            pytest.mark.post_merge,
        ],
497
        model="Qwen/Qwen2.5-VL-7B-Instruct",
498
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
499
        delayed_start=0,
500
        timeout=360,
501
502
503
504
        request_payloads=[
            chat_payload(
                [
                    {
505
506
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
507
                    },
508
509
                    {
                        "type": "image_url",
510
                        "image_url": {"url": MULTIMODAL_IMG_URL},
511
512
513
                    },
                ],
                repeat_count=1,
514
515
                expected_response=["purple"],
                max_tokens=100,
516
            ),
517
        ],
518
    ),
519
520
521
522
523
    "multimodal_agg_llava": VLLMConfig(
        name="multimodal_agg_llava",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[
524
            pytest.mark.gpu_1,
525
526
527
528
            pytest.mark.profiled_vram_gib(14.9),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                922_354_000
            ),  # KV cache cap (2x safety over min=461_176_832)
529
530
531
            pytest.mark.timeout(
                300
            ),  # ~7x observed 42.7s; 7B model loads ~48s on CI (A10G/L4)
532
            pytest.mark.nightly,
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
            # https://github.com/ai-dynamo/dynamo/issues/4501
            pytest.mark.xfail(strict=False),
        ],
        model="llava-hf/llava-1.5-7b-hf",
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        delayed_start=0,
        timeout=360,
        request_payloads=[
            # HTTP URL test
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["bus"],
                temperature=0.0,
            ),
            # String content test - verifies string → array conversion for multimodal templates
            chat_payload_default(
                repeat_count=1,
                expected_response=[],  # Just validate no error
            ),
        ],
    ),
563
    # Video multimodal tests for CI use the canonical aggregated multimodal launcher.
564
565
    "multimodal_video_agg": VLLMConfig(
        name="multimodal_video_agg",
566
        directory=vllm_dir,
567
        script_name="agg_multimodal.sh",
568
        marks=[
569
570
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
571
        ],  # TODO: profile to get max_vram and timeout
572
        model="Qwen/Qwen3-VL-2B-Instruct",
573
        delayed_start=60,  # Video models require longer loading time
574
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct"],
575
576
577
578
579
580
581
        timeout=600,  # 10 minutes for video processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
582
                        "video_url": {"url": LOCAL_VIDEO_TEST_URI},
583
584
585
                    },
                ],
                repeat_count=1,
586
                expected_response=["red", "static", "still"],
587
588
589
590
591
592
593
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_video_disagg": VLLMConfig(
        name="multimodal_video_disagg",
594
        directory=vllm_dir,
595
        script_name="disagg_multimodal_epd.sh",
596
        marks=[
597
598
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
599
        ],  # TODO: profile to get max_vram and timeout
600
        model="Qwen/Qwen3-VL-2B-Instruct",
601
        delayed_start=60,  # Video models require longer loading time
602
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
603
        timeout=600,  # 10 minutes for video processing overhead
604
605
606
607
608
609
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
610
                        "video_url": {"url": LOCAL_VIDEO_TEST_URI},
611
612
613
                    },
                ],
                repeat_count=1,
614
                expected_response=["red", "static", "still"],
615
616
                temperature=0.0,
                max_tokens=100,
617
618
            )
        ],
619
    ),
620
621
    # Audio multimodal tests for nightly CI pipeline
    # These tests validate audio inference capabilities with Qwen2-Audio model
622
623
    "multimodal_audio_agg": VLLMConfig(
        name="multimodal_audio_agg",
624
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
625
        script_name="audio_agg.sh",
626
        marks=[
627
            pytest.mark.gpu_2,  # encode worker loads Qwen2Audio on GPU (~19 GiB)
628
            pytest.mark.nightly,
629
630
            pytest.mark.timeout(600),
        ],
631
        model="Qwen/Qwen2-Audio-7B-Instruct",
632
        delayed_start=0,
633
634
635
636
637
638
639
640
641
642
643
644
645
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
                ],
                repeat_count=1,
646
647
648
649
650
651
652
653
654
655
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_audio_disagg": VLLMConfig(
        name="multimodal_audio_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="audio_disagg.sh",
656
        marks=[
657
            pytest.mark.gpu_4,  # needs 3 GPUs (encode loads Qwen2Audio ~19 GiB + prefill + decode)
658
            pytest.mark.nightly,
659
660
            pytest.mark.timeout(600),
        ],
661
        model="Qwen/Qwen2-Audio-7B-Instruct",
662
        delayed_start=0,
663
664
665
666
667
668
669
670
671
672
673
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
674
                ],
675
676
677
678
                repeat_count=1,
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
679
680
681
            )
        ],
    ),
682
683
684
685
    "aggregated_toolcalling": VLLMConfig(
        name="aggregated_toolcalling",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
686
        marks=[
687
            pytest.mark.gpu_1,  # agg_multimodal.sh uses single GPU
688
689
            pytest.mark.multimodal,
            pytest.mark.nightly,
690
        ],
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
        model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
            "--max-model-len",
            "10000",
            "--dyn-tool-call-parser",
            "hermes",
        ],
        delayed_start=0,
        timeout=600,
        request_payloads=[
            ToolCallingChatPayload(
                body={
                    "messages": [
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "text",
                                    "text": "Describe what you see in this image in detail.",
                                },
                                {
                                    "type": "image_url",
                                    "image_url": {"url": MULTIMODAL_IMG_URL},
                                },
                            ],
                        }
                    ],
                    "tools": [
                        {
                            "type": "function",
                            "function": {
                                "name": "describe_image",
                                "description": "Provides detailed description of objects and scenes in an image",
                                "parameters": {
                                    "type": "object",
                                    "properties": {
                                        "objects": {
                                            "type": "array",
                                            "items": {"type": "string"},
                                            "description": "List of objects detected in the image",
                                        },
                                        "scene": {
                                            "type": "string",
                                            "description": "Overall scene description",
                                        },
                                    },
                                    "required": ["objects", "scene"],
                                },
                            },
                        }
                    ],
                    "tool_choice": "auto",
                    "max_tokens": 1024,
                },
                repeat_count=1,
748
749
750
751
752
753
754
                expected_response=[
                    "green",
                    "purple",
                    "llm",
                    "optimize",
                    "deploy",
                ],  # OR: pass if any keyword found in tool args
755
756
757
758
759
                expected_log=[],
                expected_tool_name="describe_image",  # Validate tool call happened
            )
        ],
    ),
760
761
762
    # TODO: Enable this test case when we have 4 GPUs runners.
    # "multimodal_disagg": VLLMConfig(
    #     name="multimodal_disagg",
763
    #     directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
764
765
766
767
    #     script_name="disagg.sh",
    #     marks=[pytest.mark.gpu_4, pytest.mark.vllm],
    #     model="llava-hf/llava-1.5-7b-hf",
    #     delayed_start=45,
768
    #     script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
769
    # ),
770
771
772
773
    "completions_only": VLLMConfig(
        name="completions_only",
        directory=vllm_dir,
        script_name="agg.sh",
774
775
        marks=[
            pytest.mark.gpu_1,
776
777
778
779
            pytest.mark.profiled_vram_gib(18.3),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                4_074_898_000
            ),  # KV cache cap (2x safety over min=2_037_448_704)
780
781
782
            pytest.mark.timeout(
                420
            ),  # 7B model loads ~48s on CI (A10G/L4) vs ~15s locally
783
            pytest.mark.post_merge,
784
        ],
785
786
787
788
789
790
791
792
793
794
795
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
796
797
798
799
800
801
    "multi_node_tp_headless": VLLMConfig(
        name="multi_node_tp_headless",
        directory=os.path.join(WORKSPACE_DIR, "tests/serve"),
        script_name="multi_node_tp_headless.sh",
        marks=[
            pytest.mark.gpu_2,
802
            pytest.mark.pre_merge,
803
            # TODO: profile to get max_vram
804
805
806
807
808
809
810
811
            pytest.mark.timeout(300),
        ],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
812
813
    "guided_decoding": VLLMConfig(
        name="guided_decoding",
814
815
        directory=vllm_dir,
        script_name="agg.sh",
816
817
        marks=[
            pytest.mark.gpu_1,
818
819
820
821
            pytest.mark.profiled_vram_gib(3.8),  # actual profiled peak with kv-bytes
            pytest.mark.requested_vllm_kv_cache_bytes(
                1_119_388_000
            ),  # KV cache cap (2x safety over min=559_693_824)
822
            pytest.mark.timeout(110),  # ~5x observed 22.3s; CI machines are slower
823
824
            pytest.mark.pre_merge,
        ],
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a person with name and age",
                repeat_count=1,
                expected_response=['"name"', '"age"'],
                temperature=0.0,
                max_tokens=100,
                extra_body={
                    "guided_json": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "age": {"type": "integer"},
                        },
                        "required": ["name", "age"],
                    }
                },
843
            ),
844
845
846
847
848
849
850
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_regex": r"(red|blue|green)"},
851
            ),
852
853
854
855
856
857
858
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_choice": ["red", "blue", "green"]},
859
            ),
860
861
        ],
    ),
862
863
864
}


Alec's avatar
Alec committed
865
@pytest.fixture(params=params_with_model_mark(vllm_configs))
866
867
868
869
870
def vllm_config_test(request):
    """Fixture that provides different vLLM test configurations"""
    return vllm_configs[request.param]


871
@pytest.mark.vllm
872
@pytest.mark.e2e
Alec's avatar
Alec committed
873
def test_serve_deployment(
874
875
876
877
878
879
    vllm_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
    image_server,
Alec's avatar
Alec committed
880
):
881
882
883
    """
    Test dynamo serve deployments with different graph configurations.
    """
884
885
886
887
    config = dataclasses.replace(
        vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
888
889
890
891
892


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
893
@pytest.mark.nightly
894
895
@pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
def test_multimodal_b64(
896
897
898
899
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
900
):
901
902
903
904
905
906
    """
    Test multimodal inference with base64 url passthrough.

    This test is separate because it loads the required image at runtime
    (not collection time), ensuring it only fails when actually executed.
    """
907
908
    # Load B64 image at test execution time (uses real PNG even if MULTIMODAL_IMG is LFS pointer)
    b64_img = base64.b64encode(get_multimodal_test_image_bytes()).decode()
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939

    # Create payload with B64 image
    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["purple"],
        max_tokens=100,
    )

    # Create test config
    config = VLLMConfig(
        name="test_multimodal_b64",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        delayed_start=0,
        timeout=360,
        request_payloads=[b64_payload],
    )

940
941
942
943
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
944
945


946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.pre_merge
@pytest.mark.timeout(220)
def test_multimodal_b64_frontend_decoding(
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
):
    """
    Test multimodal inference with base64 images through frontend decoding path.

    This exercises the Rust frontend image decode + NIXL RDMA transfer path
    with inline base64 data: URIs (not HTTP URLs). Verifies that the
    strip_inline_data_urls optimization does not break correctness.
    """
    b64_img = base64.b64encode(get_multimodal_test_image_bytes()).decode()

    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["green"],
        temperature=0.0,
        max_tokens=100,
    )

    config = VLLMConfig(
        name="test_multimodal_b64_frontend_decoding",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-2B-Instruct",
            "--frontend-decoding",
        ],
        delayed_start=0,
        timeout=220,
        request_payloads=[b64_payload],
    )

    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)


1005
1006
1007
1008
1009
1010
1011
# LoRA Test Directory
lora_dir = os.path.join(vllm_dir, "launch/lora")


def lora_chat_payload(
    lora_name: str,
    s3_uri: str,
1012
    system_port: int = DefaultPort.SYSTEM1.value,
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
    repeat_count: int = 2,
    expected_response: Optional[list] = None,
    expected_log: Optional[list] = None,
    max_tokens: int = 100,
    temperature: float = 0.0,
) -> LoraTestChatPayload:
    """Create a LoRA-enabled chat payload for testing"""
    return LoraTestChatPayload(
        body={
            "model": lora_name,
            "messages": [
                {
                    "role": "user",
                    "content": "What is deep learning? Answer in one sentence.",
                }
            ],
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        },
        lora_name=lora_name,
        s3_uri=s3_uri,
        system_port=system_port,
        repeat_count=repeat_count,
        expected_response=expected_response
        or ["learning", "neural", "network", "AI", "model"],
        expected_log=expected_log or [],
    )


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
1048
@pytest.mark.post_merge
Dmitry Tokarev's avatar
Dmitry Tokarev committed
1049
@pytest.mark.skip(reason="DYN-2260")
1050
def test_lora_aggregated(
1051
1052
1053
1054
1055
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
):
    """
    Test LoRA inference with aggregated vLLM deployment.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts vLLM with LoRA support enabled
    3. Loads the LoRA adapter via system API
    4. Runs inference with the LoRA model
    """
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payload that loads LoRA and tests inference
    lora_payload = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
1072
        system_port=DefaultPort.SYSTEM1.value,
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
        repeat_count=2,
    )

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated",
        directory=vllm_dir,
        script_name="lora/agg_lora.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=minio_config.get_env_vars(),
        request_payloads=[lora_payload],
    )

1088
1089
1090
1091
1092
1093
1094
1095
1096
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config,
        request,
        ports=dynamo_dynamic_ports,
        extra_env=minio_config.get_env_vars(),
    )
1097
1098
1099
1100
1101
1102
1103


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
1104
@pytest.mark.pre_merge
1105
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
1106
def test_lora_aggregated_router(
1107
1108
1109
1110
1111
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
1112
    num_system_ports,
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
):
    """
    Test LoRA inference with aggregated vLLM deployment using KV router.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts multiple vLLM workers with LoRA support and KV router
    3. Loads the LoRA adapter on both workers via system API
    4. Runs inference with the LoRA model, verifying KV cache routing
    """
1123
1124
1125
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
1126
1127
1128
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payloads that load LoRA on both workers and test inference
1129
    # Worker 1 (DefaultPort.SYSTEM1)
1130
1131
1132
    lora_payload_worker1 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
1133
        system_port=DefaultPort.SYSTEM1.value,
1134
1135
1136
        repeat_count=1,
    )

1137
    # Worker 2 (DefaultPort.SYSTEM2)
1138
1139
1140
    lora_payload_worker2 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
1141
        system_port=DefaultPort.SYSTEM2.value,
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
        repeat_count=1,
    )

    # Additional inference payload to test routing (LoRA already loaded)
    inference_payload = chat_payload(
        content="Explain machine learning in simple terms.",
        repeat_count=2,
        expected_response=["learn", "data", "algorithm", "model", "pattern"],
        max_tokens=150,
        temperature=0.0,
    ).with_model(minio_config.lora_name)

    # Add env vars including PYTHONHASHSEED for deterministic KV event IDs
    env_vars = minio_config.get_env_vars()
    env_vars["PYTHONHASHSEED"] = "0"

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated_router",
        directory=vllm_dir,
        script_name="lora/agg_lora_router.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=env_vars,
        request_payloads=[
            lora_payload_worker1,
            lora_payload_worker2,
            inference_payload,
        ],
    )

1174
1175
1176
1177
1178
1179
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config, request, ports=dynamo_dynamic_ports, extra_env=env_vars
    )