test_vllm.py 36.3 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import base64
5
import dataclasses
6
7
import logging
import os
8
import random
9
from dataclasses import dataclass, field
10
from typing import Optional
11
12
13

import pytest

14
15
16
17
18
from tests.serve.common import (
    WORKSPACE_DIR,
    params_with_model_mark,
    run_serve_deployment,
)
19
from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes
20
from tests.serve.lora_utils import MinioLoraConfig
21
from tests.utils.constants import DefaultPort
22
23
from tests.utils.engine_process import EngineConfig
from tests.utils.payload_builder import (
24
    cached_tokens_chat_payload,
25
26
    chat_payload,
    chat_payload_default,
27
    chat_payload_with_logprobs,
28
    completion_payload_default,
29
    completion_payload_with_logprobs,
30
    metric_payload_default,
31
)
32
from tests.utils.payloads import LoraTestChatPayload, ToolCallingChatPayload
33
34
35
36

logger = logging.getLogger(__name__)


37
38
39
40
41
42
def _is_cuda13() -> bool:
    v = os.environ.get("CUDA_VERSION", "")
    # handles "13", "13.0", "13.0.1", etc.
    return v.startswith("13")


43
@dataclass
44
class VLLMConfig(EngineConfig):
45
46
    """Configuration for vLLM test scenarios"""

47
    stragglers: list[str] = field(default_factory=lambda: ["VLLM:EngineCore"])
48
49


50
vllm_dir = os.environ.get("VLLM_DIR") or os.path.join(
51
    WORKSPACE_DIR, "examples/backends/vllm"
52
)
53

54

55
# vLLM test configurations
56
# NOTE: pytest.mark.gpu_1 tests take ~5.5 minutes total to run sequentially (with models pre-cached)
57
58
59
60
# TODO: Now that these tests use dynamic ports and each config has a max_vram_gib marker,
# optimize the runtime by bin-packing multiple engine deployments in parallel on the same GPU.
# A future collector/launcher can sum max_vram_gib values to decide how many tests fit
# concurrently without exceeding available VRAM.
61
62
63
vllm_configs = {
    "aggregated": VLLMConfig(
        name="aggregated",
64
        directory=vllm_dir,
65
        script_name="agg.sh",
66
67
        marks=[
            pytest.mark.gpu_1,
68
            pytest.mark.max_vram_gib(8.6),  # observed peak 7.8 GiB (+10% safety)
69
            pytest.mark.timeout(300),  # ~7x observed 42.2s; old value before profiling
70
71
            pytest.mark.pre_merge,
        ],
72
        model="Qwen/Qwen3-0.6B",
73
74
75
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
76
77
78
79
80
81
82
83
84
85
86
            chat_payload(
                "Can you write me a song?",
                repeat_count=1,
                expected_response=["song"],
                temperature=0.0,
                max_tokens=32,
                extra_body={
                    "stop": ["song"],
                    "include_stop_str_in_output": True,
                },
            ),
87
            metric_payload_default(min_num_requests=6, backend="vllm"),
88
        ],
89
    ),
90
91
92
93
    "aggregated_logprobs": VLLMConfig(
        name="aggregated_logprobs",
        directory=vllm_dir,
        script_name="agg.sh",
94
95
96
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(8.6),  # observed peak 7.8 GiB (+10% safety)
97
            pytest.mark.timeout(120),  # ~5x observed 24.3s; CI machines are slower
98
99
            pytest.mark.post_merge,
        ],
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                top_logprobs=3,
            ),
            completion_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                logprobs=5,
            ),
        ],
    ),
118
119
120
121
    "aggregated_lmcache": VLLMConfig(
        name="aggregated_lmcache",
        directory=vllm_dir,
        script_name="agg_lmcache.sh",
122
        marks=[
123
            pytest.mark.lmcache,
124
            pytest.mark.gpu_1,
125
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.4 GiB (+10% safety)
126
            pytest.mark.timeout(360),  # ~7x observed 49.0s; old value before profiling
127
            pytest.mark.pre_merge,
128
            pytest.mark.skipif(
129
130
131
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
            ),
132
        ],
133
134
135
136
137
138
139
140
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
141
142
143
144
    "aggregated_lmcache_multiproc": VLLMConfig(
        name="aggregated_lmcache_multiproc",
        directory=vllm_dir,
        script_name="agg_lmcache_multiproc.sh",
145
        marks=[
146
            pytest.mark.lmcache,
147
            pytest.mark.gpu_1,
148
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.4 GiB (+10% safety)
149
            pytest.mark.timeout(360),  # ~7x observed 49.3s; old value before profiling
150
            pytest.mark.pre_merge,
151
            pytest.mark.skipif(
152
153
154
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
            ),
155
        ],
156
157
158
159
160
161
162
163
164
165
166
        model="Qwen/Qwen3-0.6B",
        env={
            "PROMETHEUS_MULTIPROC_DIR": f"/tmp/prometheus_multiproc_test_{os.getpid()}_{random.randint(0, 10000)}"
        },
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
167
168
169
170
    "agg-request-plane-tcp": VLLMConfig(
        name="agg-request-plane-tcp",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
171
172
        marks=[
            pytest.mark.gpu_1,
173
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.3 GiB (+10% safety)
174
            pytest.mark.timeout(300),  # ~7x observed 43.0s; old value before profiling
175
176
            pytest.mark.pre_merge,
        ],
177
178
179
180
181
182
183
184
185
186
187
        model="Qwen/Qwen3-0.6B",
        script_args=["--tcp"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
    "agg-request-plane-http": VLLMConfig(
        name="agg-request-plane-http",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
188
189
        marks=[
            pytest.mark.gpu_1,
190
            pytest.mark.max_vram_gib(8.1),  # observed peak 7.3 GiB (+10% safety)
191
            pytest.mark.timeout(300),  # ~7x observed 42.3s; old value before profiling
192
193
            pytest.mark.pre_merge,
        ],
194
195
196
197
198
199
200
        model="Qwen/Qwen3-0.6B",
        script_args=["--http"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
201
202
    "agg-router": VLLMConfig(
        name="agg-router",
203
        directory=vllm_dir,
204
        script_name="agg_router.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
205
206
        marks=[
            pytest.mark.gpu_2,
207
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
208
            pytest.mark.skip(reason="DYN-2263"),
209
        ],  # TODO: profile to get max_vram and timeout
210
        model="Qwen/Qwen3-0.6B",
211
212
213
        request_payloads=[
            chat_payload_default(
                expected_log=[
214
                    r"ZMQ listener .* received batch with \d+ events \(seq=\d+(?:, [^)]*)?\)",
215
                    r"Event processor for worker_id \d+ processing event: Stored\(",
216
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
217
218
219
220
                ]
            )
        ],
        env={
221
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_kv_router::scheduling::selector=info",
222
        },
223
    ),
224
225
226
227
    "agg-router-approx": VLLMConfig(
        name="agg-router-approx",
        directory=vllm_dir,
        script_name="agg_router_approx.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
228
229
        marks=[
            pytest.mark.gpu_2,
230
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
231
            pytest.mark.skip(reason="DYN-2264"),
232
        ],  # TODO: profile to get max_vram and timeout
233
234
235
236
237
238
239
240
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            # Test approximate KV routing (--no-kv-events mode)
            # Repeated requests should show cache-aware routing in logs
            chat_payload_default(
                repeat_count=3,
                expected_log=[
                    # Verify scheduler is selecting workers with cache awareness
241
                    r"Selected worker: worker_type=\w+, worker_id=\d+ dp_rank=.*?, logit: ",
242
243
244
245
246
247
248
249
250
251
252
253
254
255
                    # After first request, should see cached blocks being tracked
                    r"with \d+ cached blocks",
                ],
            ),
            # Also test with cached tokens payload to verify usage field
            cached_tokens_chat_payload(
                repeat_count=3,
                expected_log=[
                    # Verify routing decision shows cache hits
                    r"with \d+ cached blocks",
                ],
            ),
        ],
        env={
256
            "DYN_LOG": "dynamo_kv_router::scheduling::selector=info",
257
258
        },
    ),
259
260
    "disaggregated": VLLMConfig(
        name="disaggregated",
261
        directory=vllm_dir,
262
        script_name="disagg.sh",
263
264
265
266
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.pre_merge,
        ],  # TODO: profile to get max_vram and timeout
267
        model="Qwen/Qwen3-0.6B",
268
269
270
271
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
272
    ),
273
274
    "deepep": VLLMConfig(
        name="deepep",
275
        directory=vllm_dir,
276
        script_name="dsr1_dep.sh",
277
278
279
280
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.vllm,
            pytest.mark.h100,
281
            pytest.mark.nightly,
282
            # TODO: profile to get max_vram and timeout
283
        ],
284
        model="deepseek-ai/DeepSeek-V2-Lite",
285
        script_args=[
286
287
288
289
290
291
292
293
294
            "--model",
            "deepseek-ai/DeepSeek-V2-Lite",
            "--num-nodes",
            "1",
            "--node-rank",
            "0",
            "--gpus-per-node",
            "2",
        ],
295
        timeout=700,
296
        request_payloads=[
297
298
            chat_payload_default(),
            completion_payload_default(),
299
        ],
300
    ),
301
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements
302
303
    "multimodal_e_pd_qwen": VLLMConfig(
        name="multimodal_e_pd_qwen",
304
305
        directory=vllm_dir,
        script_name="disagg_multimodal_e_pd.sh",
306
307
308
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(24.6),  # observed peak 22.3 GiB (+10% safety)
309
            pytest.mark.timeout(340),  # ~5x observed 68.4s; 2B model loads slower on CI
310
311
            pytest.mark.pre_merge,
        ],
312
313
        model="Qwen/Qwen3-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
314
315
316
        env={
            "DYN_VLLM_EMBEDDING_TRANSFER_MODE": "nixl-write",
        },
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                # With proper prompt templating, the model actually only returns "green",
                # verified behavior with native vLLM.
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
338
339
340
341
    "multimodal_agg_frontend_decoding": VLLMConfig(
        name="multimodal_agg_frontend_decoding",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
342
        # post_merge because needs real NIXL not stub
343
344
345
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(10.2),  # observed peak 9.3 GiB (+10% safety)
346
            pytest.mark.timeout(220),  # ~5x observed 43.7s; 2B model loads slower on CI
347
348
            pytest.mark.post_merge,
        ],
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
        model="Qwen/Qwen2-VL-2B-Instruct",
        # Pass --frontend-decoding to enable Rust frontend image decoding + NIXL RDMA transfer
        script_args=[
            "--model",
            "Qwen/Qwen2-VL-2B-Instruct",
            "--frontend-decoding",
        ],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
375
376
377
378
379
380
381
    # NOTE: Pack all workers on 1 GPU for lower CI resource requirements.
    # NOTE: disagg_multimodal_epd.sh uses --kv-cache-memory-bytes=512MB for P/D
    # workers. Per vLLM CacheConfig, kv_cache_memory_bytes (when not-None) ignores
    # gpu_memory_utilization (ref: https://docs.vllm.ai/en/stable/api/vllm/config/cache/),
    # so _PROFILE_PYTEST_VRAM_FRAC_OVERRIDE has no effect. Regardless of GPU_MEM
    # fractions (0.1/0.4/0.4), the 3 workers combined consistently use ~17.6 GiB
    # total on this GPU.
382
383
    "multimodal_disagg_qwen": VLLMConfig(
        name="multimodal_disagg_qwen",
384
        directory=vllm_dir,
385
        script_name="disagg_multimodal_epd.sh",
Dmitry Tokarev's avatar
Dmitry Tokarev committed
386
387
        marks=[
            pytest.mark.gpu_1,
388
            pytest.mark.max_vram_gib(19.4),  # observed peak 17.6 GiB (+10% safety)
389
            pytest.mark.pre_merge,
Dmitry Tokarev's avatar
Dmitry Tokarev committed
390
        ],
391
        model="Qwen/Qwen3-VL-2B-Instruct",
392
393
        script_args=["--model", "Qwen/Qwen3-VL-2B-Instruct", "--single-gpu"],
        timeout=360,
394
395
396
        env={
            "DYN_ENCODE_WORKER_GPU": "0",
            "DYN_PREFILL_WORKER_GPU": "0",
397
398
            "DYN_DECODE_WORKER_GPU": "0",
            "DYN_ENCODE_GPU_MEM": "0.1",
399
            "DYN_PREFILL_GPU_MEM": "0.4",
400
            "DYN_DECODE_GPU_MEM": "0.4",
401
        },
402
403
404
        request_payloads=[
            chat_payload(
                [
405
406
407
408
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
409
410
                    {
                        "type": "image_url",
411
                        "image_url": {"url": MULTIMODAL_IMG_URL},
412
413
414
                    },
                ],
                repeat_count=1,
415
                expected_response=["green"],
416
                temperature=0.0,
417
                max_tokens=100,
418
419
            )
        ],
420
    ),
421
422
    "multimodal_agg_qwen": VLLMConfig(
        name="multimodal_agg_qwen",
423
424
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
425
426
427
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(21.6),  # observed peak 19.6 GiB (+10% safety)
428
429
430
            pytest.mark.timeout(
                360
            ),  # ~7x observed 50.0s; 7B model loads ~48s on CI (A10G/L4)
431
432
            pytest.mark.post_merge,
        ],
433
        model="Qwen/Qwen2.5-VL-7B-Instruct",
434
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
435
        delayed_start=0,
436
        timeout=360,
437
438
439
440
        request_payloads=[
            chat_payload(
                [
                    {
441
442
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
443
                    },
444
445
                    {
                        "type": "image_url",
446
                        "image_url": {"url": MULTIMODAL_IMG_URL},
447
448
449
                    },
                ],
                repeat_count=1,
450
451
                expected_response=["purple"],
                max_tokens=100,
452
            ),
453
        ],
454
    ),
455
456
457
458
459
    "multimodal_agg_llava": VLLMConfig(
        name="multimodal_agg_llava",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[
460
            pytest.mark.gpu_1,
461
            pytest.mark.max_vram_gib(18.9),  # observed peak 17.1 GiB (+10% safety)
462
463
464
            pytest.mark.timeout(
                300
            ),  # ~7x observed 42.7s; 7B model loads ~48s on CI (A10G/L4)
465
            pytest.mark.nightly,
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
            # https://github.com/ai-dynamo/dynamo/issues/4501
            pytest.mark.xfail(strict=False),
        ],
        model="llava-hf/llava-1.5-7b-hf",
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        delayed_start=0,
        timeout=360,
        request_payloads=[
            # HTTP URL test
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["bus"],
                temperature=0.0,
            ),
            # String content test - verifies string → array conversion for multimodal templates
            chat_payload_default(
                repeat_count=1,
                expected_response=[],  # Just validate no error
            ),
        ],
    ),
496
497
498
    # Video multimodal tests for nightly CI pipeline
    # These tests validate video inference capabilities with LLaVA-NeXT-Video model
    # Reference: Linear OPS-3015
499
500
    "multimodal_video_agg": VLLMConfig(
        name="multimodal_video_agg",
501
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
502
        script_name="video_agg.sh",
503
504
505
506
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
507
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
508
        delayed_start=60,  # Video models require longer loading time
509
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
        timeout=600,  # 10 minutes for video processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_video_disagg": VLLMConfig(
        name="multimodal_video_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="video_disagg.sh",
533
534
535
536
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
537
538
539
540
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        delayed_start=60,  # Video models require longer loading time
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
        timeout=600,  # 10 minutes for video processing overhead
541
542
543
544
545
546
547
548
549
550
551
552
553
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
554
555
                temperature=0.0,
                max_tokens=100,
556
557
            )
        ],
558
    ),
559
560
    # Audio multimodal tests for nightly CI pipeline
    # These tests validate audio inference capabilities with Qwen2-Audio model
561
562
    "multimodal_audio_agg": VLLMConfig(
        name="multimodal_audio_agg",
563
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
564
        script_name="audio_agg.sh",
565
566
567
568
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
569
        model="Qwen/Qwen2-Audio-7B-Instruct",
570
        delayed_start=60,  # Audio models require longer loading time
571
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
572
        timeout=600,  # 10 minutes for audio processing overhead
573
574
575
576
577
578
579
580
581
582
583
584
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
                ],
                repeat_count=1,
585
586
587
588
589
590
591
592
593
594
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_audio_disagg": VLLMConfig(
        name="multimodal_audio_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="audio_disagg.sh",
595
596
597
598
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
599
600
601
602
603
604
605
606
607
608
609
610
611
612
        model="Qwen/Qwen2-Audio-7B-Instruct",
        delayed_start=60,  # Audio models require longer loading time
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        timeout=600,  # 10 minutes for audio processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
613
                ],
614
615
616
617
                repeat_count=1,
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
618
619
620
            )
        ],
    ),
621
622
623
624
    "aggregated_toolcalling": VLLMConfig(
        name="aggregated_toolcalling",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
625
626
627
628
629
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.multimodal,
            pytest.mark.nightly,
        ],  # TODO: profile to get max_vram and timeout
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
        model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
            "--max-model-len",
            "10000",
            "--dyn-tool-call-parser",
            "hermes",
        ],
        delayed_start=0,
        timeout=600,
        request_payloads=[
            ToolCallingChatPayload(
                body={
                    "messages": [
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "text",
                                    "text": "Describe what you see in this image in detail.",
                                },
                                {
                                    "type": "image_url",
                                    "image_url": {"url": MULTIMODAL_IMG_URL},
                                },
                            ],
                        }
                    ],
                    "tools": [
                        {
                            "type": "function",
                            "function": {
                                "name": "describe_image",
                                "description": "Provides detailed description of objects and scenes in an image",
                                "parameters": {
                                    "type": "object",
                                    "properties": {
                                        "objects": {
                                            "type": "array",
                                            "items": {"type": "string"},
                                            "description": "List of objects detected in the image",
                                        },
                                        "scene": {
                                            "type": "string",
                                            "description": "Overall scene description",
                                        },
                                    },
                                    "required": ["objects", "scene"],
                                },
                            },
                        }
                    ],
                    "tool_choice": "auto",
                    "max_tokens": 1024,
                },
                repeat_count=1,
                expected_response=["purple"],  # Validate image understanding
                expected_log=[],
                expected_tool_name="describe_image",  # Validate tool call happened
            )
        ],
    ),
693
694
695
    # TODO: Enable this test case when we have 4 GPUs runners.
    # "multimodal_disagg": VLLMConfig(
    #     name="multimodal_disagg",
696
    #     directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
697
698
699
700
    #     script_name="disagg.sh",
    #     marks=[pytest.mark.gpu_4, pytest.mark.vllm],
    #     model="llava-hf/llava-1.5-7b-hf",
    #     delayed_start=45,
701
    #     script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
702
    # ),
703
704
705
706
    "completions_only": VLLMConfig(
        name="completions_only",
        directory=vllm_dir,
        script_name="agg.sh",
707
708
        marks=[
            pytest.mark.gpu_1,
709
            pytest.mark.max_vram_gib(21.9),  # observed peak 19.9 GiB (+10% safety)
710
711
712
            pytest.mark.timeout(
                420
            ),  # 7B model loads ~48s on CI (A10G/L4) vs ~15s locally
713
            pytest.mark.post_merge,
714
        ],
715
716
717
718
719
720
721
722
723
724
725
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
726
727
728
729
730
731
    "multi_node_tp_headless": VLLMConfig(
        name="multi_node_tp_headless",
        directory=os.path.join(WORKSPACE_DIR, "tests/serve"),
        script_name="multi_node_tp_headless.sh",
        marks=[
            pytest.mark.gpu_2,
732
            pytest.mark.pre_merge,
733
            # TODO: profile to get max_vram
734
735
736
737
738
739
740
741
            pytest.mark.timeout(300),
        ],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
742
743
    "guided_decoding": VLLMConfig(
        name="guided_decoding",
744
745
        directory=vllm_dir,
        script_name="agg.sh",
746
747
748
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.max_vram_gib(8.6),  # observed peak 7.8 GiB (+10% safety)
749
            pytest.mark.timeout(110),  # ~5x observed 22.3s; CI machines are slower
750
751
            pytest.mark.pre_merge,
        ],
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a person with name and age",
                repeat_count=1,
                expected_response=['"name"', '"age"'],
                temperature=0.0,
                max_tokens=100,
                extra_body={
                    "guided_json": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "age": {"type": "integer"},
                        },
                        "required": ["name", "age"],
                    }
                },
770
            ),
771
772
773
774
775
776
777
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_regex": r"(red|blue|green)"},
778
            ),
779
780
781
782
783
784
785
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_choice": ["red", "blue", "green"]},
786
            ),
787
788
        ],
    ),
789
790
791
}


Alec's avatar
Alec committed
792
@pytest.fixture(params=params_with_model_mark(vllm_configs))
793
794
795
796
797
def vllm_config_test(request):
    """Fixture that provides different vLLM test configurations"""
    return vllm_configs[request.param]


798
@pytest.mark.vllm
799
@pytest.mark.e2e
Alec's avatar
Alec committed
800
def test_serve_deployment(
801
802
803
804
805
806
    vllm_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
    image_server,
Alec's avatar
Alec committed
807
):
808
809
810
    """
    Test dynamo serve deployments with different graph configurations.
    """
811
812
813
814
    config = dataclasses.replace(
        vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
815
816
817
818
819


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
820
@pytest.mark.nightly
821
822
@pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
def test_multimodal_b64(
823
824
825
826
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
827
):
828
829
830
831
832
833
    """
    Test multimodal inference with base64 url passthrough.

    This test is separate because it loads the required image at runtime
    (not collection time), ensuring it only fails when actually executed.
    """
834
835
    # Load B64 image at test execution time (uses real PNG even if MULTIMODAL_IMG is LFS pointer)
    b64_img = base64.b64encode(get_multimodal_test_image_bytes()).decode()
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866

    # Create payload with B64 image
    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["purple"],
        max_tokens=100,
    )

    # Create test config
    config = VLLMConfig(
        name="test_multimodal_b64",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        delayed_start=0,
        timeout=360,
        request_payloads=[b64_payload],
    )

867
868
869
870
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
871
872
873
874
875
876
877
878
879


# LoRA Test Directory
lora_dir = os.path.join(vllm_dir, "launch/lora")


def lora_chat_payload(
    lora_name: str,
    s3_uri: str,
880
    system_port: int = DefaultPort.SYSTEM1.value,
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
    repeat_count: int = 2,
    expected_response: Optional[list] = None,
    expected_log: Optional[list] = None,
    max_tokens: int = 100,
    temperature: float = 0.0,
) -> LoraTestChatPayload:
    """Create a LoRA-enabled chat payload for testing"""
    return LoraTestChatPayload(
        body={
            "model": lora_name,
            "messages": [
                {
                    "role": "user",
                    "content": "What is deep learning? Answer in one sentence.",
                }
            ],
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        },
        lora_name=lora_name,
        s3_uri=s3_uri,
        system_port=system_port,
        repeat_count=repeat_count,
        expected_response=expected_response
        or ["learning", "neural", "network", "AI", "model"],
        expected_log=expected_log or [],
    )


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
916
@pytest.mark.post_merge
Dmitry Tokarev's avatar
Dmitry Tokarev committed
917
@pytest.mark.skip(reason="DYN-2260")
918
def test_lora_aggregated(
919
920
921
922
923
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
):
    """
    Test LoRA inference with aggregated vLLM deployment.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts vLLM with LoRA support enabled
    3. Loads the LoRA adapter via system API
    4. Runs inference with the LoRA model
    """
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payload that loads LoRA and tests inference
    lora_payload = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
940
        system_port=DefaultPort.SYSTEM1.value,
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
        repeat_count=2,
    )

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated",
        directory=vllm_dir,
        script_name="lora/agg_lora.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=minio_config.get_env_vars(),
        request_payloads=[lora_payload],
    )

956
957
958
959
960
961
962
963
964
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config,
        request,
        ports=dynamo_dynamic_ports,
        extra_env=minio_config.get_env_vars(),
    )
965
966
967
968
969
970
971


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
972
@pytest.mark.pre_merge
973
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
974
def test_lora_aggregated_router(
975
976
977
978
979
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
980
    num_system_ports,
981
982
983
984
985
986
987
988
989
990
):
    """
    Test LoRA inference with aggregated vLLM deployment using KV router.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts multiple vLLM workers with LoRA support and KV router
    3. Loads the LoRA adapter on both workers via system API
    4. Runs inference with the LoRA model, verifying KV cache routing
    """
991
992
993
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
994
995
996
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payloads that load LoRA on both workers and test inference
997
    # Worker 1 (DefaultPort.SYSTEM1)
998
999
1000
    lora_payload_worker1 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
1001
        system_port=DefaultPort.SYSTEM1.value,
1002
1003
1004
        repeat_count=1,
    )

1005
    # Worker 2 (DefaultPort.SYSTEM2)
1006
1007
1008
    lora_payload_worker2 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
1009
        system_port=DefaultPort.SYSTEM2.value,
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
        repeat_count=1,
    )

    # Additional inference payload to test routing (LoRA already loaded)
    inference_payload = chat_payload(
        content="Explain machine learning in simple terms.",
        repeat_count=2,
        expected_response=["learn", "data", "algorithm", "model", "pattern"],
        max_tokens=150,
        temperature=0.0,
    ).with_model(minio_config.lora_name)

    # Add env vars including PYTHONHASHSEED for deterministic KV event IDs
    env_vars = minio_config.get_env_vars()
    env_vars["PYTHONHASHSEED"] = "0"

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated_router",
        directory=vllm_dir,
        script_name="lora/agg_lora_router.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=env_vars,
        request_payloads=[
            lora_payload_worker1,
            lora_payload_worker2,
            inference_payload,
        ],
    )

1042
1043
1044
1045
1046
1047
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config, request, ports=dynamo_dynamic_ports, extra_env=env_vars
    )