test_vllm.py 36.3 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import base64
5
import dataclasses
6
7
import logging
import os
8
import random
9
from dataclasses import dataclass, field
10
from typing import Optional
11
12
13

import pytest

14
15
16
17
18
from tests.serve.common import (
    WORKSPACE_DIR,
    params_with_model_mark,
    run_serve_deployment,
)
19
from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes
20
from tests.serve.lora_utils import MinioLoraConfig
21
from tests.utils.constants import DefaultPort
22
23
from tests.utils.engine_process import EngineConfig
from tests.utils.payload_builder import (
24
    cached_tokens_chat_payload,
25
26
    chat_payload,
    chat_payload_default,
27
    chat_payload_with_logprobs,
28
    completion_payload_default,
29
    completion_payload_with_logprobs,
30
    metric_payload_default,
31
)
32
from tests.utils.payloads import LoraTestChatPayload, ToolCallingChatPayload
33
34
35
36

logger = logging.getLogger(__name__)


37
38
39
40
41
42
def _is_cuda13() -> bool:
    v = os.environ.get("CUDA_VERSION", "")
    # handles "13", "13.0", "13.0.1", etc.
    return v.startswith("13")


43
@dataclass
44
class VLLMConfig(EngineConfig):
45
46
    """Configuration for vLLM test scenarios"""

47
    stragglers: list[str] = field(default_factory=lambda: ["VLLM:EngineCore"])
48
49


50
vllm_dir = os.environ.get("VLLM_DIR") or os.path.join(
51
    WORKSPACE_DIR, "examples/backends/vllm"
52
)
53

54

55
# vLLM test configurations
56
# NOTE: pytest.mark.gpu_1 tests take ~5.5 minutes total to run sequentially (with models pre-cached)
57
58
59
60
# TODO: Now that these tests use dynamic ports and each config has a max_vram_gib marker,
# optimize the runtime by bin-packing multiple engine deployments in parallel on the same GPU.
# A future collector/launcher can sum max_vram_gib values to decide how many tests fit
# concurrently without exceeding available VRAM.
61
62
63
vllm_configs = {
    "aggregated": VLLMConfig(
        name="aggregated",
64
        directory=vllm_dir,
65
        script_name="agg.sh",
66
67
        marks=[
            pytest.mark.gpu_1,
68
            pytest.mark.max_vram_gib(8.6),  # observed peak 7.8 GiB (+10% safety)
69
            pytest.mark.timeout(300),  # ~7x observed 42.2s; old value before profiling
70
71
            pytest.mark.pre_merge,
        ],
72
        model="Qwen/Qwen3-0.6B",
73
74
75
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
76
77
78
79
80
81
82
83
84
85
86
            chat_payload(
                "Can you write me a song?",
                repeat_count=1,
                expected_response=["song"],
                temperature=0.0,
                max_tokens=32,
                extra_body={
                    "stop": ["song"],
                    "include_stop_str_in_output": True,
                },
            ),
87
            metric_payload_default(min_num_requests=6, backend="vllm"),
88
        ],
89
    ),
90
91
92
93
    "aggregated_logprobs": VLLMConfig(
        name="aggregated_logprobs",
        directory=vllm_dir,
        script_name="agg.sh",
94
95
96
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(8.6),  # observed peak 7.8 GiB (+10% safety)
97
            pytest.mark.timeout(120),  # ~5x observed 24.3s; CI machines are slower
98
99
            pytest.mark.post_merge,
        ],
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                top_logprobs=3,
            ),
            completion_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                logprobs=5,
            ),
        ],
    ),
118
119
120
121
    "aggregated_lmcache": VLLMConfig(
        name="aggregated_lmcache",
        directory=vllm_dir,
        script_name="agg_lmcache.sh",
122
        marks=[
123
            pytest.mark.lmcache,
124
            pytest.mark.gpu_1,
125
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.4 GiB (+10% safety)
126
            pytest.mark.timeout(360),  # ~7x observed 49.0s; old value before profiling
127
            pytest.mark.pre_merge,
128
            pytest.mark.skipif(
129
130
131
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
            ),
132
        ],
133
134
135
136
137
138
139
140
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
141
142
143
144
    "aggregated_lmcache_multiproc": VLLMConfig(
        name="aggregated_lmcache_multiproc",
        directory=vllm_dir,
        script_name="agg_lmcache_multiproc.sh",
145
        marks=[
146
            pytest.mark.lmcache,
147
            pytest.mark.gpu_1,
148
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.4 GiB (+10% safety)
149
            pytest.mark.timeout(360),  # ~7x observed 49.3s; old value before profiling
150
            pytest.mark.pre_merge,
151
            pytest.mark.skipif(
152
153
154
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
            ),
155
        ],
156
157
158
159
160
161
162
163
164
165
166
        model="Qwen/Qwen3-0.6B",
        env={
            "PROMETHEUS_MULTIPROC_DIR": f"/tmp/prometheus_multiproc_test_{os.getpid()}_{random.randint(0, 10000)}"
        },
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
167
168
169
170
    "agg-request-plane-tcp": VLLMConfig(
        name="agg-request-plane-tcp",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
171
172
        marks=[
            pytest.mark.gpu_1,
173
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.3 GiB (+10% safety)
174
            pytest.mark.timeout(300),  # ~7x observed 43.0s; old value before profiling
175
176
            pytest.mark.pre_merge,
        ],
177
178
179
180
181
182
183
184
185
186
187
        model="Qwen/Qwen3-0.6B",
        script_args=["--tcp"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
    "agg-request-plane-http": VLLMConfig(
        name="agg-request-plane-http",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
188
189
        marks=[
            pytest.mark.gpu_1,
190
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.3 GiB (+10% safety)
191
            pytest.mark.timeout(300),  # ~7x observed 42.3s; old value before profiling
192
193
            pytest.mark.pre_merge,
        ],
194
195
196
197
198
199
200
        model="Qwen/Qwen3-0.6B",
        script_args=["--http"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
201
202
    "agg-router": VLLMConfig(
        name="agg-router",
203
        directory=vllm_dir,
204
        script_name="agg_router.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
205
206
        marks=[
            pytest.mark.gpu_2,
207
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
208
            pytest.mark.skip(reason="DYN-2263"),
209
        ],  # TODO: profile to get max_vram and timeout
210
        model="Qwen/Qwen3-0.6B",
211
212
213
        request_payloads=[
            chat_payload_default(
                expected_log=[
214
                    r"ZMQ listener .* received batch with \d+ events \(seq=\d+(?:, [^)]*)?\)",
215
                    r"Event processor for worker_id \d+ processing event: Stored\(",
216
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
217
218
219
220
                ]
            )
        ],
        env={
221
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_kv_router::scheduling::selector=info",
222
        },
223
    ),
224
225
226
227
    "agg-router-approx": VLLMConfig(
        name="agg-router-approx",
        directory=vllm_dir,
        script_name="agg_router_approx.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
228
229
        marks=[
            pytest.mark.gpu_2,
230
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
231
            pytest.mark.skip(reason="DYN-2264"),
232
        ],  # TODO: profile to get max_vram and timeout
233
234
235
236
237
238
239
240
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            # Test approximate KV routing (--no-kv-events mode)
            # Repeated requests should show cache-aware routing in logs
            chat_payload_default(
                repeat_count=3,
                expected_log=[
                    # Verify scheduler is selecting workers with cache awareness
241
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
242
243
244
245
246
247
248
249
250
251
252
253
254
255
                    # After first request, should see cached blocks being tracked
                    r"with \d+ cached blocks",
                ],
            ),
            # Also test with cached tokens payload to verify usage field
            cached_tokens_chat_payload(
                repeat_count=3,
                expected_log=[
                    # Verify routing decision shows cache hits
                    r"with \d+ cached blocks",
                ],
            ),
        ],
        env={
256
            "DYN_LOG": "dynamo_kv_router::scheduling::selector=info",
257
258
        },
    ),
259
260
    "disaggregated": VLLMConfig(
        name="disaggregated",
261
        directory=vllm_dir,
262
        script_name="disagg.sh",
263
264
265
266
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.pre_merge,
        ],  # TODO: profile to get max_vram and timeout
267
        model="Qwen/Qwen3-0.6B",
268
269
270
271
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
272
    ),
273
274
    "deepep": VLLMConfig(
        name="deepep",
275
        directory=vllm_dir,
276
        script_name="dsr1_dep.sh",
277
278
279
280
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.vllm,
            pytest.mark.h100,
281
            pytest.mark.nightly,
282
            # TODO: profile to get max_vram and timeout
283
        ],
284
        model="deepseek-ai/DeepSeek-V2-Lite",
285
        script_args=[
286
287
288
289
290
291
292
293
294
            "--model",
            "deepseek-ai/DeepSeek-V2-Lite",
            "--num-nodes",
            "1",
            "--node-rank",
            "0",
            "--gpus-per-node",
            "2",
        ],
295
        timeout=700,
296
        request_payloads=[
297
298
            chat_payload_default(),
            completion_payload_default(),
299
        ],
300
    ),
301
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements
302
303
    "multimodal_e_pd_qwen": VLLMConfig(
        name="multimodal_e_pd_qwen",
304
305
        directory=vllm_dir,
        script_name="disagg_multimodal_e_pd.sh",
306
307
308
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(24.6),  # observed peak 22.3 GiB (+10% safety)
309
            pytest.mark.timeout(340),  # ~5x observed 68.4s; 2B model loads slower on CI
310
311
            pytest.mark.pre_merge,
        ],
312
313
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                # With proper prompt templating, the model actually only returns "green",
                # verified behavior with native vLLM.
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
335
336
337
338
    "multimodal_agg_frontend_decoding": VLLMConfig(
        name="multimodal_agg_frontend_decoding",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
339
        # post_merge because needs real NIXL not stub
340
341
342
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(10.2),  # observed peak 9.3 GiB (+10% safety)
343
            pytest.mark.timeout(220),  # ~5x observed 43.7s; 2B model loads slower on CI
344
345
            pytest.mark.post_merge,
        ],
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
        model="Qwen/Qwen2-VL-2B-Instruct",
        # Pass --frontend-decoding to enable Rust frontend image decoding + NIXL RDMA transfer
        script_args=[
            "--model",
            "Qwen/Qwen2-VL-2B-Instruct",
            "--frontend-decoding",
        ],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
372
373
374
375
376
377
378
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements.
    # NOTE: disagg_multimodal_epd.sh uses --kv-cache-memory-bytes=512MB for P/D
    # workers. Per vLLM CacheConfig, kv_cache_memory_bytes (when not-None) ignores
    # gpu_memory_utilization (ref: https://docs.vllm.ai/en/stable/api/vllm/config/cache/),
    # so _PROFILE_PYTEST_VRAM_FRAC_OVERRIDE has no effect. Regardless of GPU_MEM
    # fractions (0.1/0.4/0.4), the 3 workers combined consistently use ~17.6 GiB
    # total on this GPU.
379
380
    "multimodal_disagg_qwen": VLLMConfig(
        name="multimodal_disagg_qwen",
381
        directory=vllm_dir,
382
        script_name="disagg_multimodal_epd.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
383
384
        marks=[
            pytest.mark.gpu_1,
385
            pytest.mark.max_vram_gib(19.4),  # observed peak 17.6 GiB (+10% safety)
386
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
387
        ],
388
        model="Qwen/Qwen3-VL-2B-Instruct",
389
390
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
        timeout=360,
391
392
393
        env={
            "DYN_ENCODE_WORKER_GPU": "0",
            "DYN_PREFILL_WORKER_GPU": "0",
394
395
            "DYN_DECODE_WORKER_GPU": "0",
            "DYN_ENCODE_GPU_MEM": "0.1",
396
            "DYN_PREFILL_GPU_MEM": "0.4",
397
            "DYN_DECODE_GPU_MEM": "0.4",
398
        },
399
400
401
        request_payloads=[
            chat_payload(
                [
402
403
404
405
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
406
407
                    {
                        "type": "image_url",
408
                        "image_url": {"url": MULTIMODAL_IMG_URL},
409
410
411
                    },
                ],
                repeat_count=1,
412
                expected_response=["green"],
413
                temperature=0.0,
414
                max_tokens=100,
415
416
            )
        ],
417
    ),
418
419
    "multimodal_agg_qwen": VLLMConfig(
        name="multimodal_agg_qwen",
420
421
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
422
423
424
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(21.6),  # observed peak 19.6 GiB (+10% safety)
425
426
427
            pytest.mark.timeout(
                360
            ),  # ~7x observed 50.0s; 7B model loads ~48s on CI (A10G/L4)
428
429
            pytest.mark.post_merge,
        ],
430
        model="Qwen/Qwen2.5-VL-7B-Instruct",
431
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
432
        delayed_start=0,
433
        timeout=360,
434
435
436
437
        request_payloads=[
            chat_payload(
                [
                    {
438
439
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
440
                    },
441
442
                    {
                        "type": "image_url",
443
                        "image_url": {"url": MULTIMODAL_IMG_URL},
444
445
446
                    },
                ],
                repeat_count=1,
447
448
                expected_response=["purple"],
                max_tokens=100,
449
            ),
450
        ],
451
    ),
452
453
454
455
456
    "multimodal_agg_llava": VLLMConfig(
        name="multimodal_agg_llava",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[
457
            pytest.mark.gpu_1,
458
            pytest.mark.max_vram_gib(18.9),  # observed peak 17.1 GiB (+10% safety)
459
460
461
            pytest.mark.timeout(
                300
            ),  # ~7x observed 42.7s; 7B model loads ~48s on CI (A10G/L4)
462
            pytest.mark.nightly,
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
            # https://github.com/ai-dynamo/dynamo/issues/4501
            pytest.mark.xfail(strict=False),
        ],
        model="llava-hf/llava-1.5-7b-hf",
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        delayed_start=0,
        timeout=360,
        request_payloads=[
            # HTTP URL test
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["bus"],
                temperature=0.0,
            ),
            # String content test - verifies string → array conversion for multimodal templates
            chat_payload_default(
                repeat_count=1,
                expected_response=[],  # Just validate no error
            ),
        ],
    ),
493
494
495
    # Video multimodal tests for nightly CI pipeline
    # These tests validate video inference capabilities with LLaVA-NeXT-Video model
    # Reference: Linear OPS-3015
496
497
    "multimodal_video_agg": VLLMConfig(
        name="multimodal_video_agg",
498
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
499
        script_name="video_agg.sh",
500
501
502
503
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
504
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
505
        delayed_start=60,  # Video models require longer loading time
506
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
        timeout=600,  # 10 minutes for video processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_video_disagg": VLLMConfig(
        name="multimodal_video_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="video_disagg.sh",
530
531
532
533
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
534
535
536
537
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        delayed_start=60,  # Video models require longer loading time
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
        timeout=600,  # 10 minutes for video processing overhead
538
539
540
541
542
543
544
545
546
547
548
549
550
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
551
552
                temperature=0.0,
                max_tokens=100,
553
554
            )
        ],
555
    ),
556
557
    # Audio multimodal tests for nightly CI pipeline
    # These tests validate audio inference capabilities with Qwen2-Audio model
558
559
    "multimodal_audio_agg": VLLMConfig(
        name="multimodal_audio_agg",
560
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
561
        script_name="audio_agg.sh",
562
563
564
565
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
566
        model="Qwen/Qwen2-Audio-7B-Instruct",
567
        delayed_start=60,  # Audio models require longer loading time
568
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
569
        timeout=600,  # 10 minutes for audio processing overhead
570
571
572
573
574
575
576
577
578
579
580
581
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
                ],
                repeat_count=1,
582
583
584
585
586
587
588
589
590
591
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_audio_disagg": VLLMConfig(
        name="multimodal_audio_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="audio_disagg.sh",
592
593
594
595
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
596
597
598
599
600
601
602
603
604
605
606
607
608
609
        model="Qwen/Qwen2-Audio-7B-Instruct",
        delayed_start=60,  # Audio models require longer loading time
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        timeout=600,  # 10 minutes for audio processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
610
                ],
611
612
613
614
                repeat_count=1,
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
615
616
617
            )
        ],
    ),
618
619
620
621
    "aggregated_toolcalling": VLLMConfig(
        name="aggregated_toolcalling",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
622
623
624
625
626
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.multimodal,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
        model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
            "--max-model-len",
            "10000",
            "--dyn-tool-call-parser",
            "hermes",
        ],
        delayed_start=0,
        timeout=600,
        request_payloads=[
            ToolCallingChatPayload(
                body={
                    "messages": [
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "text",
                                    "text": "Describe what you see in this image in detail.",
                                },
                                {
                                    "type": "image_url",
                                    "image_url": {"url": MULTIMODAL_IMG_URL},
                                },
                            ],
                        }
                    ],
                    "tools": [
                        {
                            "type": "function",
                            "function": {
                                "name": "describe_image",
                                "description": "Provides detailed description of objects and scenes in an image",
                                "parameters": {
                                    "type": "object",
                                    "properties": {
                                        "objects": {
                                            "type": "array",
                                            "items": {"type": "string"},
                                            "description": "List of objects detected in the image",
                                        },
                                        "scene": {
                                            "type": "string",
                                            "description": "Overall scene description",
                                        },
                                    },
                                    "required": ["objects", "scene"],
                                },
                            },
                        }
                    ],
                    "tool_choice": "auto",
                    "max_tokens": 1024,
                },
                repeat_count=1,
                expected_response=["purple"],  # Validate image understanding
                expected_log=[],
                expected_tool_name="describe_image",  # Validate tool call happened
            )
        ],
    ),
690
691
692
    # TODO: Enable this test case when we have 4 GPUs runners.
    # "multimodal_disagg": VLLMConfig(
    #     name="multimodal_disagg",
693
    #     directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
694
695
696
697
    #     script_name="disagg.sh",
    #     marks=[pytest.mark.gpu_4, pytest.mark.vllm],
    #     model="llava-hf/llava-1.5-7b-hf",
    #     delayed_start=45,
698
    #     script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
699
    # ),
700
701
702
703
    "completions_only": VLLMConfig(
        name="completions_only",
        directory=vllm_dir,
        script_name="agg.sh",
704
705
        marks=[
            pytest.mark.gpu_1,
706
            pytest.mark.max_vram_gib(21.9),  # observed peak 19.9 GiB (+10% safety)
707
708
709
            pytest.mark.timeout(
                420
            ),  # 7B model loads ~48s on CI (A10G/L4) vs ~15s locally
710
            pytest.mark.post_merge,
711
        ],
712
713
714
715
716
717
718
719
720
721
722
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
723
724
725
726
727
728
    "multi_node_tp_headless": VLLMConfig(
        name="multi_node_tp_headless",
        directory=os.path.join(WORKSPACE_DIR, "tests/serve"),
        script_name="multi_node_tp_headless.sh",
        marks=[
            pytest.mark.gpu_2,
729
            pytest.mark.pre_merge,
730
            # TODO: profile to get max_vram
731
732
733
734
735
736
737
738
            pytest.mark.timeout(300),
        ],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
739
740
    "guided_decoding": VLLMConfig(
        name="guided_decoding",
741
742
        directory=vllm_dir,
        script_name="agg.sh",
743
744
745
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(8.6),  # observed peak 7.8 GiB (+10% safety)
746
            pytest.mark.timeout(110),  # ~5x observed 22.3s; CI machines are slower
747
748
            pytest.mark.pre_merge,
        ],
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a person with name and age",
                repeat_count=1,
                expected_response=['"name"', '"age"'],
                temperature=0.0,
                max_tokens=100,
                extra_body={
                    "guided_json": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "age": {"type": "integer"},
                        },
                        "required": ["name", "age"],
                    }
                },
767
            ),
768
769
770
771
772
773
774
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_regex": r"(red|blue|green)"},
775
            ),
776
777
778
779
780
781
782
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_choice": ["red", "blue", "green"]},
783
            ),
784
785
        ],
    ),
786
787
788
}


Alec's avatar
Alec committed
789
@pytest.fixture(params=params_with_model_mark(vllm_configs))
790
791
792
793
794
def vllm_config_test(request):
    """Fixture that provides different vLLM test configurations"""
    return vllm_configs[request.param]


795
@pytest.mark.vllm
796
@pytest.mark.e2e
Alec's avatar
Alec committed
797
def test_serve_deployment(
798
799
800
801
802
803
    vllm_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
    image_server,
Alec's avatar
Alec committed
804
):
805
806
807
    """
    Test dynamo serve deployments with different graph configurations.
    """
808
809
810
811
    config = dataclasses.replace(
        vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
812
813
814
815
816


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
817
@pytest.mark.nightly
818
819
@pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
def test_multimodal_b64(
820
821
822
823
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
824
):
825
826
827
828
829
830
    """
    Test multimodal inference with base64 url passthrough.

    This test is separate because it loads the required image at runtime
    (not collection time), ensuring it only fails when actually executed.
    """
831
832
    # Load B64 image at test execution time (uses real PNG even if MULTIMODAL_IMG is LFS pointer)
    b64_img = base64.b64encode(get_multimodal_test_image_bytes()).decode()
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863

    # Create payload with B64 image
    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["purple"],
        max_tokens=100,
    )

    # Create test config
    config = VLLMConfig(
        name="test_multimodal_b64",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        delayed_start=0,
        timeout=360,
        request_payloads=[b64_payload],
    )

864
865
866
867
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
868
869
870
871
872
873
874
875
876


# LoRA Test Directory
lora_dir = os.path.join(vllm_dir, "launch/lora")


def lora_chat_payload(
    lora_name: str,
    s3_uri: str,
877
    system_port: int = DefaultPort.SYSTEM1.value,
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
    repeat_count: int = 2,
    expected_response: Optional[list] = None,
    expected_log: Optional[list] = None,
    max_tokens: int = 100,
    temperature: float = 0.0,
) -> LoraTestChatPayload:
    """Create a LoRA-enabled chat payload for testing"""
    return LoraTestChatPayload(
        body={
            "model": lora_name,
            "messages": [
                {
                    "role": "user",
                    "content": "What is deep learning? Answer in one sentence.",
                }
            ],
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        },
        lora_name=lora_name,
        s3_uri=s3_uri,
        system_port=system_port,
        repeat_count=repeat_count,
        expected_response=expected_response
        or ["learning", "neural", "network", "AI", "model"],
        expected_log=expected_log or [],
    )


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
913
@pytest.mark.post_merge
Dmitry Tokarev's avatar
Dmitry Tokarev committed
914
@pytest.mark.skip(reason="DYN-2260")
915
def test_lora_aggregated(
916
917
918
919
920
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
):
    """
    Test LoRA inference with aggregated vLLM deployment.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts vLLM with LoRA support enabled
    3. Loads the LoRA adapter via system API
    4. Runs inference with the LoRA model
    """
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payload that loads LoRA and tests inference
    lora_payload = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
937
        system_port=DefaultPort.SYSTEM1.value,
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
        repeat_count=2,
    )

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated",
        directory=vllm_dir,
        script_name="lora/agg_lora.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=minio_config.get_env_vars(),
        request_payloads=[lora_payload],
    )

953
954
955
956
957
958
959
960
961
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config,
        request,
        ports=dynamo_dynamic_ports,
        extra_env=minio_config.get_env_vars(),
    )
962
963
964
965
966
967
968


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
969
@pytest.mark.pre_merge
970
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
971
def test_lora_aggregated_router(
972
973
974
975
976
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
977
    num_system_ports,
978
979
980
981
982
983
984
985
986
987
):
    """
    Test LoRA inference with aggregated vLLM deployment using KV router.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts multiple vLLM workers with LoRA support and KV router
    3. Loads the LoRA adapter on both workers via system API
    4. Runs inference with the LoRA model, verifying KV cache routing
    """
988
989
990
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
991
992
993
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payloads that load LoRA on both workers and test inference
994
    # Worker 1 (DefaultPort.SYSTEM1)
995
996
997
    lora_payload_worker1 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
998
        system_port=DefaultPort.SYSTEM1.value,
999
1000
1001
        repeat_count=1,
    )

1002
    # Worker 2 (DefaultPort.SYSTEM2)
1003
1004
1005
    lora_payload_worker2 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
1006
        system_port=DefaultPort.SYSTEM2.value,
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
        repeat_count=1,
    )

    # Additional inference payload to test routing (LoRA already loaded)
    inference_payload = chat_payload(
        content="Explain machine learning in simple terms.",
        repeat_count=2,
        expected_response=["learn", "data", "algorithm", "model", "pattern"],
        max_tokens=150,
        temperature=0.0,
    ).with_model(minio_config.lora_name)

    # Add env vars including PYTHONHASHSEED for deterministic KV event IDs
    env_vars = minio_config.get_env_vars()
    env_vars["PYTHONHASHSEED"] = "0"

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated_router",
        directory=vllm_dir,
        script_name="lora/agg_lora_router.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=env_vars,
        request_payloads=[
            lora_payload_worker1,
            lora_payload_worker2,
            inference_payload,
        ],
    )

1039
1040
1041
1042
1043
1044
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config, request, ports=dynamo_dynamic_ports, extra_env=env_vars
    )