test_vllm.py 29.4 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import base64
5
import dataclasses
6
7
import logging
import os
8
import random
9
from dataclasses import dataclass, field
10
from typing import Optional
11
12
13

import pytest

14
15
16
17
18
from tests.serve.common import (
    WORKSPACE_DIR,
    params_with_model_mark,
    run_serve_deployment,
)
19
from tests.serve.conftest import MULTIMODAL_IMG_PATH, MULTIMODAL_IMG_URL
20
from tests.serve.lora_utils import MinioLoraConfig
21
from tests.utils.constants import DefaultPort
22
23
from tests.utils.engine_process import EngineConfig
from tests.utils.payload_builder import (
24
    cached_tokens_chat_payload,
25
26
    chat_payload,
    chat_payload_default,
27
    chat_payload_with_logprobs,
28
    completion_payload_default,
29
    completion_payload_with_logprobs,
30
    metric_payload_default,
31
)
32
from tests.utils.payloads import LoraTestChatPayload, ToolCallingChatPayload
33
34
35
36

logger = logging.getLogger(__name__)


37
38
39
40
41
42
def _is_cuda13() -> bool:
    v = os.environ.get("CUDA_VERSION", "")
    # handles "13", "13.0", "13.0.1", etc.
    return v.startswith("13")


43
@dataclass
44
class VLLMConfig(EngineConfig):
45
46
    """Configuration for vLLM test scenarios"""

47
    stragglers: list[str] = field(default_factory=lambda: ["VLLM:EngineCore"])
48
49


50
vllm_dir = os.environ.get("VLLM_DIR") or os.path.join(
51
    WORKSPACE_DIR, "examples/backends/vllm"
52
)
53

54

55
# vLLM test configurations
56
# NOTE: pytest.mark.gpu_1 tests take ~5.5 minutes total to run sequentially (with models pre-cached)
57
58
59
60
# TODO: Now that these tests use dynamic ports, optimize the runtime by bin-packing and running
# multiple engine deployments in parallel (while keeping GPU contention under control). This may
# require annotating each config with approximate GPU RAM usage so a future collector/launcher can
# bin-pack safely.
61
62
63
vllm_configs = {
    "aggregated": VLLMConfig(
        name="aggregated",
64
        directory=vllm_dir,
65
        script_name="agg.sh",
66
67
68
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
69
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
70
        ],
71
        model="Qwen/Qwen3-0.6B",
72
73
74
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
75
76
77
78
79
80
81
82
83
84
85
            chat_payload(
                "Can you write me a song?",
                repeat_count=1,
                expected_response=["song"],
                temperature=0.0,
                max_tokens=32,
                extra_body={
                    "stop": ["song"],
                    "include_stop_str_in_output": True,
                },
            ),
86
            metric_payload_default(min_num_requests=6, backend="vllm"),
87
        ],
88
    ),
89
90
91
92
    "aggregated_logprobs": VLLMConfig(
        name="aggregated_logprobs",
        directory=vllm_dir,
        script_name="agg.sh",
93
        marks=[pytest.mark.gpu_1, pytest.mark.post_merge],
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                top_logprobs=3,
            ),
            completion_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                logprobs=5,
            ),
        ],
    ),
112
113
114
115
    "aggregated_lmcache": VLLMConfig(
        name="aggregated_lmcache",
        directory=vllm_dir,
        script_name="agg_lmcache.sh",
116
117
118
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
119
            pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
120
121
122
123
124
            pytest.mark.xfail(
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
                strict=False,
            ),
125
        ],
126
127
128
129
130
131
132
133
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
134
135
136
137
    "aggregated_lmcache_multiproc": VLLMConfig(
        name="aggregated_lmcache_multiproc",
        directory=vllm_dir,
        script_name="agg_lmcache_multiproc.sh",
138
139
        marks=[
            pytest.mark.gpu_1,
140
            pytest.mark.pre_merge,
141
            pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
142
143
144
145
146
            pytest.mark.xfail(
                _is_cuda13(),
                reason="lmcache does not support CUDA 13 as of v0.3.11",
                strict=False,
            ),
147
        ],
148
149
150
151
152
153
154
155
156
157
158
        model="Qwen/Qwen3-0.6B",
        env={
            "PROMETHEUS_MULTIPROC_DIR": f"/tmp/prometheus_multiproc_test_{os.getpid()}_{random.randint(0, 10000)}"
        },
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
159
160
161
162
    "agg-request-plane-tcp": VLLMConfig(
        name="agg-request-plane-tcp",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
163
164
165
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
166
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
167
        ],
168
169
170
171
172
173
174
175
176
177
178
        model="Qwen/Qwen3-0.6B",
        script_args=["--tcp"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
    "agg-request-plane-http": VLLMConfig(
        name="agg-request-plane-http",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
179
180
181
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
182
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
183
        ],
184
185
186
187
188
189
190
        model="Qwen/Qwen3-0.6B",
        script_args=["--http"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
191
192
    "agg-router": VLLMConfig(
        name="agg-router",
193
        directory=vllm_dir,
194
        script_name="agg_router.sh",
195
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
196
        model="Qwen/Qwen3-0.6B",
197
198
199
        request_payloads=[
            chat_payload_default(
                expected_log=[
200
                    r"ZMQ listener .* received batch with \d+ events \(seq=\d+(?:, [^)]*)?\)",
201
                    r"Event processor for worker_id \d+ processing event: Stored\(",
Yan Ru Pei's avatar
Yan Ru Pei committed
202
                    r"Selected worker: worker_id=\d+ dp_rank=.*?, logit: ",
203
204
205
206
207
208
                ]
            )
        ],
        env={
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_llm::kv_router::scheduler=info",
        },
209
    ),
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    "agg-router-approx": VLLMConfig(
        name="agg-router-approx",
        directory=vllm_dir,
        script_name="agg_router_approx.sh",
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            # Test approximate KV routing (--no-kv-events mode)
            # Repeated requests should show cache-aware routing in logs
            chat_payload_default(
                repeat_count=3,
                expected_log=[
                    # Verify scheduler is selecting workers with cache awareness
                    r"Selected worker: worker_id=\d+ dp_rank=.*?, logit: ",
                    # After first request, should see cached blocks being tracked
                    r"with \d+ cached blocks",
                ],
            ),
            # Also test with cached tokens payload to verify usage field
            cached_tokens_chat_payload(
                repeat_count=3,
                expected_log=[
                    # Verify routing decision shows cache hits
                    r"with \d+ cached blocks",
                ],
            ),
        ],
        env={
            "DYN_LOG": "dynamo_llm::kv_router::scheduler=info",
        },
    ),
241
242
    "disaggregated": VLLMConfig(
        name="disaggregated",
243
        directory=vllm_dir,
244
        script_name="disagg.sh",
245
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
246
        model="Qwen/Qwen3-0.6B",
247
248
249
250
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
251
    ),
252
253
    "deepep": VLLMConfig(
        name="deepep",
254
        directory=vllm_dir,
255
        script_name="dsr1_dep.sh",
256
257
258
259
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.vllm,
            pytest.mark.h100,
260
            pytest.mark.nightly,
261
        ],
262
        model="deepseek-ai/DeepSeek-V2-Lite",
263
        script_args=[
264
265
266
267
268
269
270
271
272
            "--model",
            "deepseek-ai/DeepSeek-V2-Lite",
            "--num-nodes",
            "1",
            "--node-rank",
            "0",
            "--gpus-per-node",
            "2",
        ],
273
        timeout=700,
274
        request_payloads=[
275
276
            chat_payload_default(),
            completion_payload_default(),
277
        ],
278
    ),
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
    "multimodal_agg_qwen2vl_2b_epd": VLLMConfig(
        name="multimodal_agg_qwen2vl_2b_epd",
        directory=vllm_dir,
        script_name="agg_multimodal_epd.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen2-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen2-VL-2B-Instruct", "--single-gpu"],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
GuanLuo's avatar
GuanLuo committed
299
300
301
                # With proper prompt templating, the model actually only returns "green",
                # verified behavior with native vLLM.
                expected_response=["green"],
302
303
304
305
306
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
307
308
    "multimodal_agg_llava_epd": VLLMConfig(
        name="multimodal_agg_llava_epd",
309
        directory=vllm_dir,
310
        script_name="agg_multimodal_epd.sh",
311
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
312
        model="llava-hf/llava-1.5-7b-hf",
313
314
315
316
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        request_payloads=[
            chat_payload(
                [
317
318
319
320
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
321
322
                    {
                        "type": "image_url",
323
                        "image_url": {"url": MULTIMODAL_IMG_URL},
324
325
326
                    },
                ],
                repeat_count=1,
327
                expected_response=["purple"],
328
                temperature=0.0,
329
                max_tokens=100,
330
331
            )
        ],
332
    ),
333
334
335
336
    "multimodal_agg_qwen_epd": VLLMConfig(
        name="multimodal_agg_qwen_epd",
        directory=vllm_dir,
        script_name="agg_multimodal_epd.sh",
337
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
338
339
340
341
342
343
344
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        delayed_start=0,
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        timeout=360,
        request_payloads=[
            chat_payload(
                [
345
346
347
348
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
349
350
                    {
                        "type": "image_url",
351
                        "image_url": {"url": MULTIMODAL_IMG_URL},
352
353
354
                    },
                ],
                repeat_count=1,
355
356
                expected_response=["purple"],
                max_tokens=100,
357
358
359
            )
        ],
    ),
360
361
    "multimodal_agg_qwen": VLLMConfig(
        name="multimodal_agg_qwen",
362
363
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
364
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
365
        model="Qwen/Qwen2.5-VL-7B-Instruct",
366
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
367
        delayed_start=0,
368
        timeout=360,
369
370
371
372
        request_payloads=[
            chat_payload(
                [
                    {
373
374
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
375
                    },
376
377
                    {
                        "type": "image_url",
378
                        "image_url": {"url": MULTIMODAL_IMG_URL},
379
380
381
                    },
                ],
                repeat_count=1,
382
383
                expected_response=["purple"],
                max_tokens=100,
384
            ),
385
        ],
386
    ),
387
388
389
390
391
    "multimodal_agg_llava": VLLMConfig(
        name="multimodal_agg_llava",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[
392
            pytest.mark.gpu_1,
393
            pytest.mark.nightly,
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
            # https://github.com/ai-dynamo/dynamo/issues/4501
            pytest.mark.xfail(strict=False),
        ],
        model="llava-hf/llava-1.5-7b-hf",
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        delayed_start=0,
        timeout=360,
        request_payloads=[
            # HTTP URL test
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["bus"],
                temperature=0.0,
            ),
            # String content test - verifies string → array conversion for multimodal templates
            chat_payload_default(
                repeat_count=1,
                expected_response=[],  # Just validate no error
            ),
        ],
    ),
424
    # TODO: Update this test case when we have video multimodal support in vllm official components
425
426
    "multimodal_video_agg": VLLMConfig(
        name="multimodal_video_agg",
427
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
428
        script_name="video_agg.sh",
429
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
430
431
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        delayed_start=0,
432
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
433
        timeout=360,
434
435
436
437
438
439
440
441
442
443
444
445
446
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
447
                temperature=0.7,
448
449
            )
        ],
450
    ),
451
452
453
454
    "multimodal_audio_agg": VLLMConfig(
        name="multimodal_audio_agg",
        directory="/workspace/examples/multimodal",
        script_name="audio_agg.sh",
455
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
        model="Qwen/Qwen2-Audio-7B-Instruct",
        delayed_start=0,
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        timeout=500,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=[
                    "The original content of this audio is:'yet these thoughts affected Hester Pynne less with hope than apprehension.'"
                ],
                temperature=0.8,
            )
        ],
    ),
479
480
481
482
    "aggregated_toolcalling": VLLMConfig(
        name="aggregated_toolcalling",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
483
        marks=[pytest.mark.gpu_2, pytest.mark.multimodal, pytest.mark.nightly],
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
        model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
            "--max-model-len",
            "10000",
            "--dyn-tool-call-parser",
            "hermes",
        ],
        delayed_start=0,
        timeout=600,
        request_payloads=[
            ToolCallingChatPayload(
                body={
                    "messages": [
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "text",
                                    "text": "Describe what you see in this image in detail.",
                                },
                                {
                                    "type": "image_url",
                                    "image_url": {"url": MULTIMODAL_IMG_URL},
                                },
                            ],
                        }
                    ],
                    "tools": [
                        {
                            "type": "function",
                            "function": {
                                "name": "describe_image",
                                "description": "Provides detailed description of objects and scenes in an image",
                                "parameters": {
                                    "type": "object",
                                    "properties": {
                                        "objects": {
                                            "type": "array",
                                            "items": {"type": "string"},
                                            "description": "List of objects detected in the image",
                                        },
                                        "scene": {
                                            "type": "string",
                                            "description": "Overall scene description",
                                        },
                                    },
                                    "required": ["objects", "scene"],
                                },
                            },
                        }
                    ],
                    "tool_choice": "auto",
                    "max_tokens": 1024,
                },
                repeat_count=1,
                expected_response=["purple"],  # Validate image understanding
                expected_log=[],
                expected_tool_name="describe_image",  # Validate tool call happened
            )
        ],
    ),
547
548
549
    # TODO: Enable this test case when we have 4 GPUs runners.
    # "multimodal_disagg": VLLMConfig(
    #     name="multimodal_disagg",
550
    #     directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
551
552
553
554
    #     script_name="disagg.sh",
    #     marks=[pytest.mark.gpu_4, pytest.mark.vllm],
    #     model="llava-hf/llava-1.5-7b-hf",
    #     delayed_start=45,
555
    #     script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
556
    # ),
557
558
559
560
    "completions_only": VLLMConfig(
        name="completions_only",
        directory=vllm_dir,
        script_name="agg.sh",
561
562
        marks=[
            pytest.mark.gpu_1,
563
            pytest.mark.post_merge,
564
565
566
            pytest.mark.timeout(
                420
            ),  # 3x estimated time (60s) + download time (240s) for 7B model
567
        ],
568
569
570
571
572
573
574
575
576
577
578
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
579
580
    "guided_decoding": VLLMConfig(
        name="guided_decoding",
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
        directory=vllm_dir,
        script_name="agg.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a person with name and age",
                repeat_count=1,
                expected_response=['"name"', '"age"'],
                temperature=0.0,
                max_tokens=100,
                extra_body={
                    "guided_json": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "age": {"type": "integer"},
                        },
                        "required": ["name", "age"],
                    }
                },
602
            ),
603
604
605
606
607
608
609
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_regex": r"(red|blue|green)"},
610
            ),
611
612
613
614
615
616
617
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_choice": ["red", "blue", "green"]},
618
            ),
619
620
        ],
    ),
621
622
623
}


Alec's avatar
Alec committed
624
@pytest.fixture(params=params_with_model_mark(vllm_configs))
625
626
627
628
629
def vllm_config_test(request):
    """Fixture that provides different vLLM test configurations"""
    return vllm_configs[request.param]


630
@pytest.mark.vllm
631
@pytest.mark.e2e
Alec's avatar
Alec committed
632
def test_serve_deployment(
633
634
635
636
637
638
    vllm_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
    image_server,
Alec's avatar
Alec committed
639
):
640
641
642
    """
    Test dynamo serve deployments with different graph configurations.
    """
643
644
645
646
    config = dataclasses.replace(
        vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
647
648
649
650
651


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
652
@pytest.mark.nightly
653
654
@pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
def test_multimodal_b64(
655
656
657
658
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
659
):
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
    """
    Test multimodal inference with base64 url passthrough.

    This test is separate because it loads the required image at runtime
    (not collection time), ensuring it only fails when actually executed.
    """
    # Load B64 image at test execution time
    with open(MULTIMODAL_IMG_PATH, "rb") as f:
        b64_img = base64.b64encode(f.read()).decode()

    # Create payload with B64 image
    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["purple"],
        max_tokens=100,
    )

    # Create test config
    config = VLLMConfig(
        name="test_multimodal_b64",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        delayed_start=0,
        timeout=360,
        request_payloads=[b64_payload],
    )

700
701
702
703
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
704
705
706
707
708
709
710
711
712


# LoRA Test Directory
lora_dir = os.path.join(vllm_dir, "launch/lora")


def lora_chat_payload(
    lora_name: str,
    s3_uri: str,
713
    system_port: int = DefaultPort.SYSTEM1.value,
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
    repeat_count: int = 2,
    expected_response: Optional[list] = None,
    expected_log: Optional[list] = None,
    max_tokens: int = 100,
    temperature: float = 0.0,
) -> LoraTestChatPayload:
    """Create a LoRA-enabled chat payload for testing"""
    return LoraTestChatPayload(
        body={
            "model": lora_name,
            "messages": [
                {
                    "role": "user",
                    "content": "What is deep learning? Answer in one sentence.",
                }
            ],
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        },
        lora_name=lora_name,
        s3_uri=s3_uri,
        system_port=system_port,
        repeat_count=repeat_count,
        expected_response=expected_response
        or ["learning", "neural", "network", "AI", "model"],
        expected_log=expected_log or [],
    )


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
749
@pytest.mark.post_merge
750
def test_lora_aggregated(
751
752
753
754
755
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
):
    """
    Test LoRA inference with aggregated vLLM deployment.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts vLLM with LoRA support enabled
    3. Loads the LoRA adapter via system API
    4. Runs inference with the LoRA model
    """
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payload that loads LoRA and tests inference
    lora_payload = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
772
        system_port=DefaultPort.SYSTEM1.value,
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
        repeat_count=2,
    )

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated",
        directory=vllm_dir,
        script_name="lora/agg_lora.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=minio_config.get_env_vars(),
        request_payloads=[lora_payload],
    )

788
789
790
791
792
793
794
795
796
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config,
        request,
        ports=dynamo_dynamic_ports,
        extra_env=minio_config.get_env_vars(),
    )
797
798
799
800
801
802
803


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
804
@pytest.mark.post_merge
805
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
806
def test_lora_aggregated_router(
807
808
809
810
811
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
812
    num_system_ports,
813
814
815
816
817
818
819
820
821
822
):
    """
    Test LoRA inference with aggregated vLLM deployment using KV router.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts multiple vLLM workers with LoRA support and KV router
    3. Loads the LoRA adapter on both workers via system API
    4. Runs inference with the LoRA model, verifying KV cache routing
    """
823
824
825
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
826
827
828
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payloads that load LoRA on both workers and test inference
829
    # Worker 1 (DefaultPort.SYSTEM1)
830
831
832
    lora_payload_worker1 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
833
        system_port=DefaultPort.SYSTEM1.value,
834
835
836
        repeat_count=1,
    )

837
    # Worker 2 (DefaultPort.SYSTEM2)
838
839
840
    lora_payload_worker2 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
841
        system_port=DefaultPort.SYSTEM2.value,
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
        repeat_count=1,
    )

    # Additional inference payload to test routing (LoRA already loaded)
    inference_payload = chat_payload(
        content="Explain machine learning in simple terms.",
        repeat_count=2,
        expected_response=["learn", "data", "algorithm", "model", "pattern"],
        max_tokens=150,
        temperature=0.0,
    ).with_model(minio_config.lora_name)

    # Add env vars including PYTHONHASHSEED for deterministic KV event IDs
    env_vars = minio_config.get_env_vars()
    env_vars["PYTHONHASHSEED"] = "0"

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated_router",
        directory=vllm_dir,
        script_name="lora/agg_lora_router.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=env_vars,
        request_payloads=[
            lora_payload_worker1,
            lora_payload_worker2,
            inference_payload,
        ],
    )

874
875
876
877
878
879
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config, request, ports=dynamo_dynamic_ports, extra_env=env_vars
    )