test_vllm.py 29.3 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
3
# SPDX-License-Identifier: Apache-2.0

4
import base64
5
import dataclasses
6
7
import logging
import os
8
import random
9
from dataclasses import dataclass, field
10
from typing import Optional
11
12
13

import pytest

14
15
16
17
18
from tests.serve.common import (
    WORKSPACE_DIR,
    params_with_model_mark,
    run_serve_deployment,
)
19
from tests.serve.conftest import MULTIMODAL_IMG_PATH, MULTIMODAL_IMG_URL
20
from tests.serve.lora_utils import MinioLoraConfig
21
from tests.utils.constants import DefaultPort
22
23
from tests.utils.engine_process import EngineConfig
from tests.utils.payload_builder import (
24
    cached_tokens_chat_payload,
25
26
    chat_payload,
    chat_payload_default,
27
    chat_payload_with_logprobs,
28
    completion_payload_default,
29
    completion_payload_with_logprobs,
30
    metric_payload_default,
31
)
32
from tests.utils.payloads import LoraTestChatPayload, ToolCallingChatPayload
33
34
35
36
37

logger = logging.getLogger(__name__)


@dataclass
38
class VLLMConfig(EngineConfig):
39
40
    """Configuration for vLLM test scenarios"""

41
    stragglers: list[str] = field(default_factory=lambda: ["VLLM:EngineCore"])
42
43


44
vllm_dir = os.environ.get("VLLM_DIR") or os.path.join(
45
    WORKSPACE_DIR, "examples/backends/vllm"
46
)
47

48

49
# vLLM test configurations
50
# NOTE: pytest.mark.gpu_1 tests take ~5.5 minutes total to run sequentially (with models pre-cached)
51
52
53
54
# TODO: Now that these tests use dynamic ports, optimize the runtime by bin-packing and running
# multiple engine deployments in parallel (while keeping GPU contention under control). This may
# require annotating each config with approximate GPU RAM usage so a future collector/launcher can
# bin-pack safely.
55
56
57
vllm_configs = {
    "aggregated": VLLMConfig(
        name="aggregated",
58
        directory=vllm_dir,
59
        script_name="agg.sh",
60
61
62
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
63
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
64
        ],
65
        model="Qwen/Qwen3-0.6B",
66
67
68
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
69
70
71
72
73
74
75
76
77
78
79
            chat_payload(
                "Can you write me a song?",
                repeat_count=1,
                expected_response=["song"],
                temperature=0.0,
                max_tokens=32,
                extra_body={
                    "stop": ["song"],
                    "include_stop_str_in_output": True,
                },
            ),
80
            metric_payload_default(min_num_requests=6, backend="vllm"),
81
        ],
82
    ),
83
84
85
86
    "aggregated_logprobs": VLLMConfig(
        name="aggregated_logprobs",
        directory=vllm_dir,
        script_name="agg.sh",
87
        marks=[pytest.mark.gpu_1, pytest.mark.post_merge],
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                top_logprobs=3,
            ),
            completion_payload_with_logprobs(
                repeat_count=2,
                expected_response=["AI", "knock", "joke"],
                max_tokens=30,
                temperature=0.0,
                logprobs=5,
            ),
        ],
    ),
106
107
108
109
    "aggregated_lmcache": VLLMConfig(
        name="aggregated_lmcache",
        directory=vllm_dir,
        script_name="agg_lmcache.sh",
110
111
112
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
113
            pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
114
        ],
115
116
117
118
119
120
121
122
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
123
124
125
126
    "aggregated_lmcache_multiproc": VLLMConfig(
        name="aggregated_lmcache_multiproc",
        directory=vllm_dir,
        script_name="agg_lmcache_multiproc.sh",
127
128
        marks=[
            pytest.mark.gpu_1,
129
            pytest.mark.pre_merge,
130
            pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
131
        ],
132
133
134
135
136
137
138
139
140
141
142
        model="Qwen/Qwen3-0.6B",
        env={
            "PROMETHEUS_MULTIPROC_DIR": f"/tmp/prometheus_multiproc_test_{os.getpid()}_{random.randint(0, 10000)}"
        },
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
            metric_payload_default(min_num_requests=6, backend="vllm"),
            metric_payload_default(min_num_requests=6, backend="lmcache"),
        ],
    ),
143
144
145
146
    "agg-request-plane-tcp": VLLMConfig(
        name="agg-request-plane-tcp",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
147
148
149
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
150
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
151
        ],
152
153
154
155
156
157
158
159
160
161
162
        model="Qwen/Qwen3-0.6B",
        script_args=["--tcp"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
    "agg-request-plane-http": VLLMConfig(
        name="agg-request-plane-http",
        directory=vllm_dir,
        script_name="agg_request_planes.sh",
163
164
165
        marks=[
            pytest.mark.gpu_1,
            pytest.mark.pre_merge,
166
            pytest.mark.timeout(300),  # 3x measured time (43s) + download time (150s)
167
        ],
168
169
170
171
172
173
174
        model="Qwen/Qwen3-0.6B",
        script_args=["--http"],
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
    ),
175
176
    "agg-router": VLLMConfig(
        name="agg-router",
177
        directory=vllm_dir,
178
        script_name="agg_router.sh",
179
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
180
        model="Qwen/Qwen3-0.6B",
181
182
183
        request_payloads=[
            chat_payload_default(
                expected_log=[
184
                    r"ZMQ listener .* received batch with \d+ events \(seq=\d+(?:, [^)]*)?\)",
185
                    r"Event processor for worker_id \d+ processing event: Stored\(",
Yan Ru Pei's avatar
Yan Ru Pei committed
186
                    r"Selected worker: worker_id=\d+ dp_rank=.*?, logit: ",
187
188
189
190
191
192
                ]
            )
        ],
        env={
            "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_llm::kv_router::scheduler=info",
        },
193
    ),
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
    "agg-router-approx": VLLMConfig(
        name="agg-router-approx",
        directory=vllm_dir,
        script_name="agg_router_approx.sh",
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            # Test approximate KV routing (--no-kv-events mode)
            # Repeated requests should show cache-aware routing in logs
            chat_payload_default(
                repeat_count=3,
                expected_log=[
                    # Verify scheduler is selecting workers with cache awareness
                    r"Selected worker: worker_id=\d+ dp_rank=.*?, logit: ",
                    # After first request, should see cached blocks being tracked
                    r"with \d+ cached blocks",
                ],
            ),
            # Also test with cached tokens payload to verify usage field
            cached_tokens_chat_payload(
                repeat_count=3,
                expected_log=[
                    # Verify routing decision shows cache hits
                    r"with \d+ cached blocks",
                ],
            ),
        ],
        env={
            "DYN_LOG": "dynamo_llm::kv_router::scheduler=info",
        },
    ),
225
226
    "disaggregated": VLLMConfig(
        name="disaggregated",
227
        directory=vllm_dir,
228
        script_name="disagg.sh",
229
        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
230
        model="Qwen/Qwen3-0.6B",
231
232
233
234
        request_payloads=[
            chat_payload_default(),
            completion_payload_default(),
        ],
235
    ),
236
237
    "deepep": VLLMConfig(
        name="deepep",
238
        directory=vllm_dir,
239
        script_name="dsr1_dep.sh",
240
241
242
243
        marks=[
            pytest.mark.gpu_2,
            pytest.mark.vllm,
            pytest.mark.h100,
244
            pytest.mark.nightly,
245
        ],
246
        model="deepseek-ai/DeepSeek-V2-Lite",
247
        script_args=[
248
249
250
251
252
253
254
255
256
            "--model",
            "deepseek-ai/DeepSeek-V2-Lite",
            "--num-nodes",
            "1",
            "--node-rank",
            "0",
            "--gpus-per-node",
            "2",
        ],
257
        timeout=700,
258
        request_payloads=[
259
260
            chat_payload_default(),
            completion_payload_default(),
261
        ],
262
    ),
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    "multimodal_agg_qwen2vl_2b_epd": VLLMConfig(
        name="multimodal_agg_qwen2vl_2b_epd",
        directory=vllm_dir,
        script_name="agg_multimodal_epd.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen2-VL-2B-Instruct",
        script_args=["--model", "Qwen/Qwen2-VL-2B-Instruct", "--single-gpu"],
        request_payloads=[
            chat_payload(
                [
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": MULTIMODAL_IMG_URL},
                    },
                ],
                repeat_count=1,
                expected_response=["purple"],
                temperature=0.0,
                max_tokens=100,
            )
        ],
    ),
289
290
    "multimodal_agg_llava_epd": VLLMConfig(
        name="multimodal_agg_llava_epd",
291
        directory=vllm_dir,
292
        script_name="agg_multimodal_epd.sh",
293
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
294
        model="llava-hf/llava-1.5-7b-hf",
295
296
297
298
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        request_payloads=[
            chat_payload(
                [
299
300
301
302
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
303
304
                    {
                        "type": "image_url",
305
                        "image_url": {"url": MULTIMODAL_IMG_URL},
306
307
308
                    },
                ],
                repeat_count=1,
309
                expected_response=["purple"],
310
                temperature=0.0,
311
                max_tokens=100,
312
313
            )
        ],
314
    ),
315
316
317
318
    "multimodal_agg_qwen_epd": VLLMConfig(
        name="multimodal_agg_qwen_epd",
        directory=vllm_dir,
        script_name="agg_multimodal_epd.sh",
319
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
320
321
322
323
324
325
326
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        delayed_start=0,
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        timeout=360,
        request_payloads=[
            chat_payload(
                [
327
328
329
330
                    {
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
                    },
331
332
                    {
                        "type": "image_url",
333
                        "image_url": {"url": MULTIMODAL_IMG_URL},
334
335
336
                    },
                ],
                repeat_count=1,
337
338
                expected_response=["purple"],
                max_tokens=100,
339
340
341
            )
        ],
    ),
342
343
    "multimodal_agg_qwen": VLLMConfig(
        name="multimodal_agg_qwen",
344
345
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
346
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
347
        model="Qwen/Qwen2.5-VL-7B-Instruct",
348
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
349
        delayed_start=0,
350
        timeout=360,
351
352
353
354
        request_payloads=[
            chat_payload(
                [
                    {
355
356
                        "type": "text",
                        "text": "What colors are in the following image? Respond only with the colors.",
357
                    },
358
359
                    {
                        "type": "image_url",
360
                        "image_url": {"url": MULTIMODAL_IMG_URL},
361
362
363
                    },
                ],
                repeat_count=1,
364
365
                expected_response=["purple"],
                max_tokens=100,
366
            ),
367
        ],
368
    ),
369
370
371
372
373
    "multimodal_agg_llava": VLLMConfig(
        name="multimodal_agg_llava",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[
374
            pytest.mark.gpu_1,
375
            pytest.mark.nightly,
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
            # https://github.com/ai-dynamo/dynamo/issues/4501
            pytest.mark.xfail(strict=False),
        ],
        model="llava-hf/llava-1.5-7b-hf",
        script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
        delayed_start=0,
        timeout=360,
        request_payloads=[
            # HTTP URL test
            chat_payload(
                [
                    {"type": "text", "text": "What is in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "http://images.cocodataset.org/test2017/000000155781.jpg"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["bus"],
                temperature=0.0,
            ),
            # String content test - verifies string → array conversion for multimodal templates
            chat_payload_default(
                repeat_count=1,
                expected_response=[],  # Just validate no error
            ),
        ],
    ),
406
    # TODO: Update this test case when we have video multimodal support in vllm official components
407
408
    "multimodal_video_agg": VLLMConfig(
        name="multimodal_video_agg",
409
        directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
410
        script_name="video_agg.sh",
411
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
412
413
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        delayed_start=0,
414
        script_args=["--model", "llava-hf/LLaVA-NeXT-Video-7B-hf"],
415
        timeout=360,
416
417
418
419
420
421
422
423
424
425
426
427
428
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "Describe the video in detail"},
                    {
                        "type": "video_url",
                        "video_url": {
                            "url": "https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=["rabbit"],
429
                temperature=0.7,
430
431
            )
        ],
432
    ),
433
434
435
436
    "multimodal_audio_agg": VLLMConfig(
        name="multimodal_audio_agg",
        directory="/workspace/examples/multimodal",
        script_name="audio_agg.sh",
437
        marks=[pytest.mark.gpu_2, pytest.mark.nightly],
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
        model="Qwen/Qwen2-Audio-7B-Instruct",
        delayed_start=0,
        script_args=["--model", "Qwen/Qwen2-Audio-7B-Instruct"],
        timeout=500,
        request_payloads=[
            chat_payload(
                [
                    {"type": "text", "text": "What is recited in the audio?"},
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/main/datasets/mini_en/wav/1221-135766-0002.wav"
                        },
                    },
                ],
                repeat_count=1,
                expected_response=[
                    "The original content of this audio is:'yet these thoughts affected Hester Pynne less with hope than apprehension.'"
                ],
                temperature=0.8,
            )
        ],
    ),
461
462
463
464
    "aggregated_toolcalling": VLLMConfig(
        name="aggregated_toolcalling",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
465
        marks=[pytest.mark.gpu_2, pytest.mark.multimodal, pytest.mark.nightly],
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
        model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
        script_args=[
            "--model",
            "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
            "--max-model-len",
            "10000",
            "--dyn-tool-call-parser",
            "hermes",
        ],
        delayed_start=0,
        timeout=600,
        request_payloads=[
            ToolCallingChatPayload(
                body={
                    "messages": [
                        {
                            "role": "user",
                            "content": [
                                {
                                    "type": "text",
                                    "text": "Describe what you see in this image in detail.",
                                },
                                {
                                    "type": "image_url",
                                    "image_url": {"url": MULTIMODAL_IMG_URL},
                                },
                            ],
                        }
                    ],
                    "tools": [
                        {
                            "type": "function",
                            "function": {
                                "name": "describe_image",
                                "description": "Provides detailed description of objects and scenes in an image",
                                "parameters": {
                                    "type": "object",
                                    "properties": {
                                        "objects": {
                                            "type": "array",
                                            "items": {"type": "string"},
                                            "description": "List of objects detected in the image",
                                        },
                                        "scene": {
                                            "type": "string",
                                            "description": "Overall scene description",
                                        },
                                    },
                                    "required": ["objects", "scene"],
                                },
                            },
                        }
                    ],
                    "tool_choice": "auto",
                    "max_tokens": 1024,
                },
                repeat_count=1,
                expected_response=["purple"],  # Validate image understanding
                expected_log=[],
                expected_tool_name="describe_image",  # Validate tool call happened
            )
        ],
    ),
529
530
531
    # TODO: Enable this test case when we have 4 GPUs runners.
    # "multimodal_disagg": VLLMConfig(
    #     name="multimodal_disagg",
532
    #     directory=os.path.join(WORKSPACE_DIR, "examples/multimodal"),
533
534
535
536
    #     script_name="disagg.sh",
    #     marks=[pytest.mark.gpu_4, pytest.mark.vllm],
    #     model="llava-hf/llava-1.5-7b-hf",
    #     delayed_start=45,
537
    #     script_args=["--model", "llava-hf/llava-1.5-7b-hf"],
538
    # ),
539
540
541
542
    "completions_only": VLLMConfig(
        name="completions_only",
        directory=vllm_dir,
        script_name="agg.sh",
543
544
        marks=[
            pytest.mark.gpu_1,
545
            pytest.mark.post_merge,
546
547
548
            pytest.mark.timeout(
                420
            ),  # 3x estimated time (60s) + download time (240s) for 7B model
549
        ],
550
551
552
553
554
555
556
557
558
559
560
        model="deepseek-ai/deepseek-llm-7b-base",
        script_args=[
            "--model",
            "deepseek-ai/deepseek-llm-7b-base",
            "--dyn-endpoint-types",
            "completions",
        ],
        request_payloads=[
            completion_payload_default(),
        ],
    ),
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
    "guided_decoding_json": VLLMConfig(
        name="guided_decoding_json",
        directory=vllm_dir,
        script_name="agg.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a person with name and age",
                repeat_count=1,
                expected_response=['"name"', '"age"'],
                temperature=0.0,
                max_tokens=100,
                extra_body={
                    "guided_json": {
                        "type": "object",
                        "properties": {
                            "name": {"type": "string"},
                            "age": {"type": "integer"},
                        },
                        "required": ["name", "age"],
                    }
                },
            )
        ],
    ),
    "guided_decoding_regex": VLLMConfig(
        name="guided_decoding_regex",
        directory=vllm_dir,
        script_name="agg.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_regex": r"(red|blue|green)"},
            )
        ],
    ),
    "guided_decoding_choice": VLLMConfig(
        name="guided_decoding_choice",
        directory=vllm_dir,
        script_name="agg.sh",
        marks=[pytest.mark.gpu_1, pytest.mark.pre_merge],
        model="Qwen/Qwen3-0.6B",
        request_payloads=[
            chat_payload(
                "Generate a color name (red, blue, or green)",
                repeat_count=1,
                expected_response=["red", "blue", "green"],
                temperature=0.0,
                max_tokens=20,
                extra_body={"guided_choice": ["red", "blue", "green"]},
            )
        ],
    ),
621
622
623
}


Alec's avatar
Alec committed
624
@pytest.fixture(params=params_with_model_mark(vllm_configs))
625
626
627
628
629
def vllm_config_test(request):
    """Fixture that provides different vLLM test configurations"""
    return vllm_configs[request.param]


630
@pytest.mark.vllm
631
@pytest.mark.e2e
Alec's avatar
Alec committed
632
def test_serve_deployment(
633
634
635
636
637
638
    vllm_config_test,
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
    image_server,
Alec's avatar
Alec committed
639
):
640
641
642
    """
    Test dynamo serve deployments with different graph configurations.
    """
643
644
645
646
    config = dataclasses.replace(
        vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
647
648
649
650
651


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
652
@pytest.mark.nightly
653
654
@pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
def test_multimodal_b64(
655
656
657
658
    request,
    runtime_services_dynamic_ports,
    dynamo_dynamic_ports,
    predownload_models,
659
):
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
    """
    Test multimodal inference with base64 url passthrough.

    This test is separate because it loads the required image at runtime
    (not collection time), ensuring it only fails when actually executed.
    """
    # Load B64 image at test execution time
    with open(MULTIMODAL_IMG_PATH, "rb") as f:
        b64_img = base64.b64encode(f.read()).decode()

    # Create payload with B64 image
    b64_payload = chat_payload(
        [
            {
                "type": "text",
                "text": "What colors are in the following image? Respond only with the colors.",
            },
            {
                "type": "image_url",
                "image_url": {"url": f"data:image/png;base64,{b64_img}"},
            },
        ],
        repeat_count=1,
        expected_response=["purple"],
        max_tokens=100,
    )

    # Create test config
    config = VLLMConfig(
        name="test_multimodal_b64",
        directory=vllm_dir,
        script_name="agg_multimodal.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen2.5-VL-7B-Instruct",
        script_args=["--model", "Qwen/Qwen2.5-VL-7B-Instruct"],
        delayed_start=0,
        timeout=360,
        request_payloads=[b64_payload],
    )

700
701
702
703
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(config, request, ports=dynamo_dynamic_ports)
704
705
706
707
708
709
710
711
712


# LoRA Test Directory
lora_dir = os.path.join(vllm_dir, "launch/lora")


def lora_chat_payload(
    lora_name: str,
    s3_uri: str,
713
    system_port: int = DefaultPort.SYSTEM1.value,
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
    repeat_count: int = 2,
    expected_response: Optional[list] = None,
    expected_log: Optional[list] = None,
    max_tokens: int = 100,
    temperature: float = 0.0,
) -> LoraTestChatPayload:
    """Create a LoRA-enabled chat payload for testing"""
    return LoraTestChatPayload(
        body={
            "model": lora_name,
            "messages": [
                {
                    "role": "user",
                    "content": "What is deep learning? Answer in one sentence.",
                }
            ],
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        },
        lora_name=lora_name,
        s3_uri=s3_uri,
        system_port=system_port,
        repeat_count=repeat_count,
        expected_response=expected_response
        or ["learning", "neural", "network", "AI", "model"],
        expected_log=expected_log or [],
    )


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_1
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
749
@pytest.mark.post_merge
750
def test_lora_aggregated(
751
752
753
754
755
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
):
    """
    Test LoRA inference with aggregated vLLM deployment.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts vLLM with LoRA support enabled
    3. Loads the LoRA adapter via system API
    4. Runs inference with the LoRA model
    """
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payload that loads LoRA and tests inference
    lora_payload = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
772
        system_port=DefaultPort.SYSTEM1.value,
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
        repeat_count=2,
    )

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated",
        directory=vllm_dir,
        script_name="lora/agg_lora.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=minio_config.get_env_vars(),
        request_payloads=[lora_payload],
    )

788
789
790
791
792
793
794
795
796
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config,
        request,
        ports=dynamo_dynamic_ports,
        extra_env=minio_config.get_env_vars(),
    )
797
798
799
800
801
802
803


@pytest.mark.vllm
@pytest.mark.e2e
@pytest.mark.gpu_2
@pytest.mark.model("Qwen/Qwen3-0.6B")
@pytest.mark.timeout(600)
804
@pytest.mark.post_merge
805
@pytest.mark.parametrize("num_system_ports", [2], indirect=True)
806
def test_lora_aggregated_router(
807
808
809
810
811
    request,
    runtime_services_dynamic_ports,
    predownload_models,
    minio_lora_service,
    dynamo_dynamic_ports,
812
    num_system_ports,
813
814
815
816
817
818
819
820
821
822
):
    """
    Test LoRA inference with aggregated vLLM deployment using KV router.

    This test:
    1. Uses MinIO fixture to provide S3-compatible storage with uploaded LoRA
    2. Starts multiple vLLM workers with LoRA support and KV router
    3. Loads the LoRA adapter on both workers via system API
    4. Runs inference with the LoRA model, verifying KV cache routing
    """
823
824
825
    assert (
        num_system_ports >= 2
    ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2"
826
827
828
    minio_config: MinioLoraConfig = minio_lora_service

    # Create payloads that load LoRA on both workers and test inference
829
    # Worker 1 (DefaultPort.SYSTEM1)
830
831
832
    lora_payload_worker1 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
833
        system_port=DefaultPort.SYSTEM1.value,
834
835
836
        repeat_count=1,
    )

837
    # Worker 2 (DefaultPort.SYSTEM2)
838
839
840
    lora_payload_worker2 = lora_chat_payload(
        lora_name=minio_config.lora_name,
        s3_uri=minio_config.get_s3_uri(),
841
        system_port=DefaultPort.SYSTEM2.value,
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
        repeat_count=1,
    )

    # Additional inference payload to test routing (LoRA already loaded)
    inference_payload = chat_payload(
        content="Explain machine learning in simple terms.",
        repeat_count=2,
        expected_response=["learn", "data", "algorithm", "model", "pattern"],
        max_tokens=150,
        temperature=0.0,
    ).with_model(minio_config.lora_name)

    # Add env vars including PYTHONHASHSEED for deterministic KV event IDs
    env_vars = minio_config.get_env_vars()
    env_vars["PYTHONHASHSEED"] = "0"

    # Create test config with MinIO environment variables
    config = VLLMConfig(
        name="test_lora_aggregated_router",
        directory=vllm_dir,
        script_name="lora/agg_lora_router.sh",
        marks=[],  # markers at function-level
        model="Qwen/Qwen3-0.6B",
        timeout=600,
        env=env_vars,
        request_payloads=[
            lora_payload_worker1,
            lora_payload_worker2,
            inference_payload,
        ],
    )

874
875
876
877
878
879
    config = dataclasses.replace(
        config, frontend_port=dynamo_dynamic_ports.frontend_port
    )
    run_serve_deployment(
        config, request, ports=dynamo_dynamic_ports, extra_env=env_vars
    )