test_vllm.py 33.5 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import base64
5
import dataclasses
6
7
import logging
import os
8
import random
9
from dataclasses import dataclass, field
10
from typing import Optional
11
12
13

import pytest

14
15
16
17
18
from tests.serve.common import (
    WORKSPACE_DIR,
    params_with_model_mark,
    run_serve_deployment,
)
19
from tests.serve.conftest import MULTIMODAL_IMG_URL, get_multimodal_test_image_bytes
20
from tests.serve.lora_utils import MinioLoraConfig
21
from tests.utils.constants import DefaultPort
22
23
from tests.utils.engine_process import EngineConfig
from tests.utils.payload_builder import (
24
    cached_tokens_chat_payload,
25
26
    chat_payload,
    chat_payload_default,
27
    chat_payload_with_logprobs,
28
    completion_payload_default,
29
    completion_payload_with_logprobs,
30
    metric_payload_default,
31
)
32
from tests.utils.payloads import LoraTestChatPayload, ToolCallingChatPayload
33
34
35
36

logger = logging.getLogger(__name__)


37
38
39
40
41
42
def _is_cuda13() -> bool:
    v = os.environ.get("CUDA_VERSION", "")
    # handles "13", "13.0", "13.0.1", etc.
    return v.startswith("13")


43
@dataclass
44
class VLLMConfig(EngineConfig):
45
46
    """Configuration for vLLM test scenarios"""

47
    stragglers: list[str] = field(default_factory=lambda: ["VLLM:EngineCore"])
48
49


50
vllm_dir = os.environ.get("VLLM_DIR") or os.path.join(
51
    WORKSPACE_DIR, "examples/backends/vllm"
52
)
53

54

55
# vLLM test configurations
56
# NOTE: pytest.mark.gpu_1 tests take ~5.5 minutes total to run sequentially (with models pre-cached)
57
58
59
60
# TODO: Now that these tests use dynamic ports, optimize the runtime by bin-packing and running
# multiple engine deployments in parallel (while keeping GPU contention under control). This may
# require annotating each config with approximate GPU RAM usage so a future collector/launcher can
# bin-pack safely.
61
62
63
vllm_configs = {
    "aggregated": VLLMConfig(
        name="aggregated",
64
        directory=vllm_dir,
65
        script_name="agg.sh",
66
67
68
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
69
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
70
        ],
71
        model="Qwen/Qwen3-0.6B",
72
73
74
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
75
76
77
78
79
80
81
82
83
84
85
            chat_payload(
                "Can you write me a song?",
                repeat_count=1,
                expected_response=["song"],
                temperature=0.0,
                max_tokens=32,
                extra_body={
                    "stop": ["song"],
                    "include_stop_str_in_output": True,
                },
            ),
86
            metric_payload_default(min_num_requests=6, backend="vllm"),
87
        ],
88
    ),
89
90
91
92
    "aggregated_logprobs": VLLMConfig(
        name="aggregated_logprobs",
        directory=vllm_dir,
        script_name="agg.sh",
93
        marks=[pytest.mark.gpu_1, pytest.mark.post_merge],
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                top_logprobs=3,
            ),
            completion_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                logprobs=5,
            ),
        ],
    ),
112
113
114
115
    "aggregated_lmcache": VLLMConfig(
        name="aggregated_lmcache",
        directory=vllm_dir,
        script_name="agg_lmcache.sh",
116
117
118
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
119
            pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
120
121
122
123
124
            pytest.mark.xfail(
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
                strict=False,
            ),
125
        ],
126
127
128
129
130
131
132
133
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
134
135
136
137
    "aggregated_lmcache_multiproc": VLLMConfig(
        name="aggregated_lmcache_multiproc",
        directory=vllm_dir,
        script_name="agg_lmcache_multiproc.sh",
138
139
        marks=[
            pytest.mark.gpu_1,
140
            pytest.mark.pre_merge,
141
            pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
142
143
144
145
146
            pytest.mark.xfail(
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
                strict=False,
            ),
147
        ],
148
149
150
151
152
153
154
155
156
157
158
        model="Qwen/Qwen3-0.6B",
        env={
            "PROMETHEUS_MULTIPROC_DIR": f"/tmp/prometheus_multiproc_test_{os.getpid()}_{random.randint(0, 10000)}"
        },
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
159
160
161
162
    "agg-request-plane-tcp": VLLMConfig(
        name="agg-request-plane-tcp",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
163
164
165
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
166
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
167
        ],
168
169
170
171
172
173
174
175
176
177
178
        model="Qwen/Qwen3-0.6B",
        script_args=["--tcp"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
    "agg-request-plane-http": VLLMConfig(
        name="agg-request-plane-http",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
179
180
181
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
182
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
183
        ],
184
185
186
187
188
189
190
        model="Qwen/Qwen3-0.6B",
        script_args=["--http"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
191
192
    "agg-router": VLLMConfig(
        name="agg-router",
193
        directory=vllm_dir,
194
        script_name="agg_router.sh",
195
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
196
        model="Qwen/Qwen3-0.6B",
197
198
199
        request_payloads=[
            chat_payload_default(
                expected_log=[
200
                    r"ZMQ listener .* received batch with \d+ events \(seq=\d+(?:, [^)]*)?\)",
201
                    r"Event processor for worker_id \d+ processing event: Stored\(",
Yan Ru Pei's avatar
Yan Ru Pei committed
202
                    r"Selected worker: worker_id=\d+ dp_rank=.*?, logit: ",
203
204
205
206
207
208
                ]
            )
        ],
        env={
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_llm::kv_router::scheduler=info",
        },
209
    ),
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    "agg-router-approx": VLLMConfig(
        name="agg-router-approx",
        directory=vllm_dir,
        script_name="agg_router_approx.sh",
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            # Test approximate KV routing (--no-kv-events mode)
            # Repeated requests should show cache-aware routing in logs
            chat_payload_default(
                repeat_count=3,
                expected_log=[
                    # Verify scheduler is selecting workers with cache awareness
                    r"Selected worker: worker_id=\d+ dp_rank=.*?, logit: ",
                    # After first request, should see cached blocks being tracked
                    r"with \d+ cached blocks",
                ],
            ),
            # Also test with cached tokens payload to verify usage field
            cached_tokens_chat_payload(
                repeat_count=3,
                expected_log=[
                    # Verify routing decision shows cache hits
                    r"with \d+ cached blocks",
                ],
            ),
        ],
        env={
            "DYN_LOG": "dynamo_llm::kv_router::scheduler=info",
        },
    ),
241
242
    "disaggregated": VLLMConfig(
        name="disaggregated",
243
        directory=vllm_dir,
244
        script_name="disagg.sh",
245
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
246
        model="Qwen/Qwen3-0.6B",
247
248
249
250
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
251
    ),
252
253
    "deepep": VLLMConfig(
        name="deepep",
254
        directory=vllm_dir,
255
        script_name="dsr1_dep.sh",
256
257
258
259
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.vllm,
            pytest.mark.h100,
260
            pytest.mark.nightly,
261
        ],
262
        model="deepseek-ai/DeepSeek-V2-Lite",
263
        script_args=[
264
265
266
267
268
269
270
271
272
            "--model",
            "deepseek-ai/DeepSeek-V2-Lite",
            "--num-nodes",
            "1",
            "--node-rank",
            "0",
            "--gpus-per-node",
            "2",
        ],
273
        timeout=700,
274
        request_payloads=[
275
276
            chat_payload_default(),
            completion_payload_default(),
277
        ],
278
    ),
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
    "multimodal_disagg_qwen2vl_2b_e_pd": VLLMConfig(
        name="multimodal_disagg_qwen2vl_2b_e_pd",
        directory=vllm_dir,
        script_name="disagg_multimodal_e_pd.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen2-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen2-VL-2B-Instruct", "--single-gpu"],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                # With proper prompt templating, the model actually only returns "green",
                # verified behavior with native vLLM.
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    "multimodal_agg_frontend_decoding": VLLMConfig(
        name="multimodal_agg_frontend_decoding",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen2-VL-2B-Instruct",
        # Pass --frontend-decoding to enable Rust frontend image decoding + NIXL RDMA transfer
        script_args=[
            "--model",
            "Qwen/Qwen2-VL-2B-Instruct",
            "--frontend-decoding",
        ],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                expected_response=["green"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
338
339
    "multimodal_agg_llava_epd": VLLMConfig(
        name="multimodal_agg_llava_epd",
340
        directory=vllm_dir,
341
        script_name="agg_multimodal_epd.sh",
342
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
343
        model="llava-hf/llava-1.5-7b-hf",
344
345
346
347
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        request_payloads=[
            chat_payload(
                [
348
349
350
351
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
352
353
                    {
                        "type": "image_url",
354
                        "image_url": {"url": MULTIMODAL_IMG_URL},
355
356
357
                    },
                ],
                repeat_count=1,
358
                expected_response=["purple"],
359
                temperature=0.0,
360
                max_tokens=100,
361
362
            )
        ],
363
    ),
364
365
366
367
    "multimodal_agg_qwen_epd": VLLMConfig(
        name="multimodal_agg_qwen_epd",
        directory=vllm_dir,
        script_name="agg_multimodal_epd.sh",
368
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
369
370
371
372
373
374
375
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        delayed_start=0,
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        timeout=360,
        request_payloads=[
            chat_payload(
                [
376
377
378
379
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
380
381
                    {
                        "type": "image_url",
382
                        "image_url": {"url": MULTIMODAL_IMG_URL},
383
384
385
                    },
                ],
                repeat_count=1,
386
387
                expected_response=["purple"],
                max_tokens=100,
388
389
390
            )
        ],
    ),
391
392
    "multimodal_agg_qwen": VLLMConfig(
        name="multimodal_agg_qwen",
393
394
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
395
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
396
        model="Qwen/Qwen2.5-VL-7B-Instruct",
397
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
398
        delayed_start=0,
399
        timeout=360,
400
401
402
403
        request_payloads=[
            chat_payload(
                [
                    {
404
405
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
406
                    },
407
408
                    {
                        "type": "image_url",
409
                        "image_url": {"url": MULTIMODAL_IMG_URL},
410
411
412
                    },
                ],
                repeat_count=1,
413
414
                expected_response=["purple"],
                max_tokens=100,
415
            ),
416
        ],
417
    ),
418
419
420
421
422
    "multimodal_agg_llava": VLLMConfig(
        name="multimodal_agg_llava",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[
423
            pytest.mark.gpu_1,
424
            pytest.mark.nightly,
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
            # https://github.com/ai-dynamo/dynamo/issues/4501
            pytest.mark.xfail(strict=False),
        ],
        model="llava-hf/llava-1.5-7b-hf",
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        delayed_start=0,
        timeout=360,
        request_payloads=[
            # HTTP URL test
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["bus"],
                temperature=0.0,
            ),
            # String content test - verifies string → array conversion for multimodal templates
            chat_payload_default(
                repeat_count=1,
                expected_response=[],  # Just validate no error
            ),
        ],
    ),
455
456
457
    # Video multimodal tests for nightly CI pipeline
    # These tests validate video inference capabilities with LLaVA-NeXT-Video model
    # Reference: Linear OPS-3015
458
459
    "multimodal_video_agg": VLLMConfig(
        name="multimodal_video_agg",
460
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
461
        script_name="video_agg.sh",
462
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
463
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
464
        delayed_start=60,  # Video models require longer loading time
465
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
        timeout=600,  # 10 minutes for video processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_video_disagg": VLLMConfig(
        name="multimodal_video_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="video_disagg.sh",
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        delayed_start=60,  # Video models require longer loading time
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
        timeout=600,  # 10 minutes for video processing overhead
494
495
496
497
498
499
500
501
502
503
504
505
506
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
507
508
                temperature=0.0,
                max_tokens=100,
509
510
            )
        ],
511
    ),
512
513
    # Audio multimodal tests for nightly CI pipeline
    # These tests validate audio inference capabilities with Qwen2-Audio model
514
515
    "multimodal_audio_agg": VLLMConfig(
        name="multimodal_audio_agg",
516
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
517
        script_name="audio_agg.sh",
518
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
519
        model="Qwen/Qwen2-Audio-7B-Instruct",
520
        delayed_start=60,  # Audio models require longer loading time
521
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
522
        timeout=600,  # 10 minutes for audio processing overhead
523
524
525
526
527
528
529
530
531
532
533
534
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
                ],
                repeat_count=1,
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
    "multimodal_audio_disagg": VLLMConfig(
        name="multimodal_audio_disagg",
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
        script_name="audio_disagg.sh",
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
        model="Qwen/Qwen2-Audio-7B-Instruct",
        delayed_start=60,  # Audio models require longer loading time
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        timeout=600,  # 10 minutes for audio processing overhead
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
560
                ],
561
562
563
564
                repeat_count=1,
                expected_response=["Hester", "Pynne"],
                temperature=0.0,
                max_tokens=100,
565
566
567
            )
        ],
    ),
568
569
570
571
    "aggregated_toolcalling": VLLMConfig(
        name="aggregated_toolcalling",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
572
        marks=[pytest.mark.gpu_2, pytest.mark.multimodal, pytest.mark.nightly],
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
        model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
            "--max-model-len",
            "10000",
            "--dyn-tool-call-parser",
            "hermes",
        ],
        delayed_start=0,
        timeout=600,
        request_payloads=[
            ToolCallingChatPayload(
                body={
                    "messages": [
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "text",
                                    "text": "Describe what you see in this image in detail.",
                                },
                                {
                                    "type": "image_url",
                                    "image_url": {"url": MULTIMODAL_IMG_URL},
                                },
                            ],
                        }
                    ],
                    "tools": [
                        {
                            "type": "function",
                            "function": {
                                "name": "describe_image",
                                "description": "Provides detailed description of objects and scenes in an image",
                                "parameters": {
                                    "type": "object",
                                    "properties": {
                                        "objects": {
                                            "type": "array",
                                            "items": {"type": "string"},
                                            "description": "List of objects detected in the image",
                                        },
                                        "scene": {
                                            "type": "string",
                                            "description": "Overall scene description",
                                        },
                                    },
                                    "required": ["objects", "scene"],
                                },
                            },
                        }
                    ],
                    "tool_choice": "auto",
                    "max_tokens": 1024,
                },
                repeat_count=1,
                expected_response=["purple"],  # Validate image understanding
                expected_log=[],
                expected_tool_name="describe_image",  # Validate tool call happened
            )
        ],
    ),
636
637
638
    # TODO: Enable this test case when we have 4 GPUs runners.
    # "multimodal_disagg": VLLMConfig(
    #     name="multimodal_disagg",
639
    #     directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
640
641
642
643
    #     script_name="disagg.sh",
    #     marks=[pytest.mark.gpu_4, pytest.mark.vllm],
    #     model="llava-hf/llava-1.5-7b-hf",
    #     delayed_start=45,
644
    #     script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
645
    # ),
646
647
648
649
    "completions_only": VLLMConfig(
        name="completions_only",
        directory=vllm_dir,
        script_name="agg.sh",
650
651
        marks=[
            pytest.mark.gpu_1,
652
            pytest.mark.post_merge,
653
654
655
            pytest.mark.timeout(
                420
            ),  # 3x estimated time (60s) + download time (240s) for 7B model
656
        ],
657
658
659
660
661
662
663
664
665
666
667
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
    "multi_node_tp_headless": VLLMConfig(
        name="multi_node_tp_headless",
        directory=os.path.join(WORKSPACE_DIR, "tests/serve"),
        script_name="multi_node_tp_headless.sh",
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.post_merge,
            pytest.mark.timeout(300),
        ],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
683
684
    "guided_decoding": VLLMConfig(
        name="guided_decoding",
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
        directory=vllm_dir,
        script_name="agg.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a person with name and age",
                repeat_count=1,
                expected_response=['"name"', '"age"'],
                temperature=0.0,
                max_tokens=100,
                extra_body={
                    "guided_json": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "age": {"type": "integer"},
                        },
                        "required": ["name", "age"],
                    }
                },
706
            ),
707
708
709
710
711
712
713
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_regex": r"(red|blue|green)"},
714
            ),
715
716
717
718
719
720
721
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_choice": ["red", "blue", "green"]},
722
            ),
723
724
        ],
    ),
725
726
727
}


Alec's avatar
Alec committed
728
@pytest.fixture(params=params_with_model_mark(vllm_configs))
729
730
731
732
733
def vllm_config_test(request):
    """Fixture that provides different vLLM test configurations"""
    return vllm_configs[request.param]


734
@pytest.mark.vllm
735
@pytest.mark.e2e
Alec's avatar
Alec committed
736
def test_serve_deployment(
737
738
739
740
741
742
    vllm_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
    image_server,
Alec's avatar
Alec committed
743
):
744
745
746
    """
    Test dynamo serve deployments with different graph configurations.
    """
747
748
749
750
    config = dataclasses.replace(
        vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
751
752
753
754
755


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
756
@pytest.mark.nightly
757
758
@pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
def test_multimodal_b64(
759
760
761
762
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
763
):
764
765
766
767
768
769
    """
    Test multimodal inference with base64 url passthrough.

    This test is separate because it loads the required image at runtime
    (not collection time), ensuring it only fails when actually executed.
    """
770
771
    # Load B64 image at test execution time (uses real PNG even if MULTIMODAL_IMG is LFS pointer)
    b64_img = base64.b64encode(get_multimodal_test_image_bytes()).decode()
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802

    # Create payload with B64 image
    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["purple"],
        max_tokens=100,
    )

    # Create test config
    config = VLLMConfig(
        name="test_multimodal_b64",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        delayed_start=0,
        timeout=360,
        request_payloads=[b64_payload],
    )

803
804
805
806
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
807
808
809
810
811
812
813
814
815


# LoRA Test Directory
lora_dir = os.path.join(vllm_dir, "launch/lora")


def lora_chat_payload(
    lora_name: str,
    s3_uri: str,
816
    system_port: int = DefaultPort.SYSTEM1.value,
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
    repeat_count: int = 2,
    expected_response: Optional[list] = None,
    expected_log: Optional[list] = None,
    max_tokens: int = 100,
    temperature: float = 0.0,
) -> LoraTestChatPayload:
    """Create a LoRA-enabled chat payload for testing"""
    return LoraTestChatPayload(
        body={
            "model": lora_name,
            "messages": [
                {
                    "role": "user",
                    "content": "What is deep learning? Answer in one sentence.",
                }
            ],
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        },
        lora_name=lora_name,
        s3_uri=s3_uri,
        system_port=system_port,
        repeat_count=repeat_count,
        expected_response=expected_response
        or ["learning", "neural", "network", "AI", "model"],
        expected_log=expected_log or [],
    )


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
852
@pytest.mark.post_merge
853
def test_lora_aggregated(
854
855
856
857
858
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
):
    """
    Test LoRA inference with aggregated vLLM deployment.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts vLLM with LoRA support enabled
    3. Loads the LoRA adapter via system API
    4. Runs inference with the LoRA model
    """
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payload that loads LoRA and tests inference
    lora_payload = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
875
        system_port=DefaultPort.SYSTEM1.value,
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
        repeat_count=2,
    )

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated",
        directory=vllm_dir,
        script_name="lora/agg_lora.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=minio_config.get_env_vars(),
        request_payloads=[lora_payload],
    )

891
892
893
894
895
896
897
898
899
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config,
        request,
        ports=dynamo_dynamic_ports,
        extra_env=minio_config.get_env_vars(),
    )
900
901
902
903
904
905
906


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
907
@pytest.mark.post_merge
908
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
909
def test_lora_aggregated_router(
910
911
912
913
914
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
915
    num_system_ports,
916
917
918
919
920
921
922
923
924
925
):
    """
    Test LoRA inference with aggregated vLLM deployment using KV router.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts multiple vLLM workers with LoRA support and KV router
    3. Loads the LoRA adapter on both workers via system API
    4. Runs inference with the LoRA model, verifying KV cache routing
    """
926
927
928
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
929
930
931
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payloads that load LoRA on both workers and test inference
932
    # Worker 1 (DefaultPort.SYSTEM1)
933
934
935
    lora_payload_worker1 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
936
        system_port=DefaultPort.SYSTEM1.value,
937
938
939
        repeat_count=1,
    )

940
    # Worker 2 (DefaultPort.SYSTEM2)
941
942
943
    lora_payload_worker2 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
944
        system_port=DefaultPort.SYSTEM2.value,
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
        repeat_count=1,
    )

    # Additional inference payload to test routing (LoRA already loaded)
    inference_payload = chat_payload(
        content="Explain machine learning in simple terms.",
        repeat_count=2,
        expected_response=["learn", "data", "algorithm", "model", "pattern"],
        max_tokens=150,
        temperature=0.0,
    ).with_model(minio_config.lora_name)

    # Add env vars including PYTHONHASHSEED for deterministic KV event IDs
    env_vars = minio_config.get_env_vars()
    env_vars["PYTHONHASHSEED"] = "0"

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated_router",
        directory=vllm_dir,
        script_name="lora/agg_lora_router.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=env_vars,
        request_payloads=[
            lora_payload_worker1,
            lora_payload_worker2,
            inference_payload,
        ],
    )

977
978
979
980
981
982
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config, request, ports=dynamo_dynamic_ports, extra_env=env_vars
    )