test_router_e2e_with_mockers.py 41 KB
Newer Older
1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
5
6
7
8

# Parallelization: Hermetic tests (xdist-safe via dynamic ports + per-test namespaces).
# Tested on: Linux container.
# Combined pre_merge wall time (this file):
# - Serialized: 304.01s.
# - Parallel (-n auto): 34.55s (269.46s saved, 8.80x).
9
10
11
12
13
14
#
# NOTE: TCP request plane is NOT tested here. These tests use --num-workers > 1 which spawns
# multiple workers in a single process sharing one TCP server. The shared TCP server uses
# endpoint_path (e.g., "generate") as the routing key, causing handler collisions when multiple
# workers register the same endpoint. This is a test-only limitation; production deployments
# with separate processes per worker work correctly with TCP.
15
import asyncio
16
17
import logging
import os
18
from typing import Any, Dict, Optional
19

20
import aiohttp
21
22
import pytest

23
from tests.router.common import (  # utilities
24
    _test_busy_threshold_endpoint,
25
26
27
    _test_python_router_bindings,
    _test_router_basic,
    _test_router_decisions,
28
    _test_router_decisions_disagg,
29
30
31
32
33
34
35
    _test_router_indexers_sync,
    _test_router_overload_503,
    _test_router_query_instance_id,
    _test_router_two_routers,
    generate_random_suffix,
    get_runtime,
)
Alec's avatar
Alec committed
36
from tests.utils.constants import ROUTER_MODEL_NAME
37
from tests.utils.managed_process import ManagedProcess
38
from tests.utils.port_utils import allocate_ports, deallocate_ports
39

40
41
42
43
logger = logging.getLogger(__name__)

MODEL_NAME = ROUTER_MODEL_NAME

44
45
46
47
pytestmark = [
    pytest.mark.pre_merge,
    pytest.mark.gpu_0,
    pytest.mark.integration,
48
    pytest.mark.model(MODEL_NAME),
49
    pytest.mark.skip(reason="DYN-2365 - Flaky, temporarily disabled"),
50
]
51
52
NUM_MOCKERS = 2
SPEEDUP_RATIO = 10.0
53
54
55
BASE_PORT = 9100  # Base port for general test allocations (frontend, system, etc.)
BASE_PORT_BOOTSTRAP = 10100  # Base port for disagg bootstrap rendezvous
BASE_PORT_ZMQ = 11100  # Base port for ZMQ KV event publishing
56
NUM_REQUESTS = 100
57
BLOCK_SIZE = 16
58
59


60
def get_unique_ports(
61
62
63
64
    request,
    num_ports: int = 1,
    store_backend: str = "etcd",
    request_plane: str = "nats",
65
    registration_order: str = "prefill_first",
66
) -> list[int]:
67
    """Allocate random free ports for xdist-safe router tests.
68

69
70
71
    This replaces the previous "test-name offset" scheme with the shared flock-backed
    allocator from `tests.utils.port_utils`, which avoids collisions across pytest-xdist
    worker processes.
72

73
74
75
76
    Notes:
    - The extra parameters are kept for call-site compatibility (they no longer affect
      the chosen ports).
    - Ports are released at the end of the test via a pytest finalizer.
77
    """
78
79
80
    _ = (store_backend, request_plane, registration_order)
    ports = allocate_ports(num_ports, BASE_PORT)
    request.addfinalizer(lambda: deallocate_ports(ports))
81
82
83
    return ports


84
85
86
87
88
89
90
91
92
93
94
95
96
# Shared test payload for all tests
TEST_PAYLOAD: Dict[str, Any] = {
    "model": MODEL_NAME,
    "messages": [
        {
            "role": "user",
            "content": "In a quiet meadow tucked between rolling hills, a plump gray rabbit nibbled on clover beneath the shade of a gnarled oak tree. Its ears twitched at the faint rustle of leaves, but it remained calm, confident in the safety of its burrow just a few hops away. The late afternoon sun warmed its fur, and tiny dust motes danced in the golden light as bees hummed lazily nearby. Though the rabbit lived a simple life, every day was an adventure of scents, shadows, and snacks—an endless search for the tastiest patch of greens and the softest spot to nap.",
        }
    ],
    "stream": True,
    "max_tokens": 10,
}

97

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def _build_mocker_command(
    endpoint: str,
    store_backend: str,
    num_workers: int,
    mocker_args: Dict[str, Any],
    worker_type: Optional[str] = None,
) -> list[str]:
    """Build the mocker CLI command with all arguments.

    Args:
        endpoint: The dynamo endpoint string
        store_backend: Storage backend ("etcd" or "file")
        num_workers: Number of workers to spawn (uses --num-workers flag)
        mocker_args: Dictionary of mocker arguments
        worker_type: Optional worker type ("prefill" or "decode") for disagg mode

    Returns:
        List of command arguments for subprocess
    """
    command = [
        "python",
        "-m",
        "dynamo.mocker",
        "--model-path",
        MODEL_NAME,
        "--endpoint",
        endpoint,
125
        "--discovery-backend",
126
127
128
129
130
131
132
        store_backend,
        "--num-workers",
        str(num_workers),
    ]

    # Add worker type flag for disaggregated mode
    if worker_type == "prefill":
133
        command.extend(["--disaggregation-mode", "prefill"])
134
    elif worker_type == "decode":
135
        command.extend(["--disaggregation-mode", "decode"])
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

    # Add individual CLI arguments from mocker_args
    if "speedup_ratio" in mocker_args:
        command.extend(["--speedup-ratio", str(mocker_args["speedup_ratio"])])
    if "block_size" in mocker_args:
        command.extend(["--block-size", str(mocker_args["block_size"])])
    if "num_gpu_blocks" in mocker_args:
        command.extend(
            ["--num-gpu-blocks-override", str(mocker_args["num_gpu_blocks"])]
        )
    if "max_num_seqs" in mocker_args:
        command.extend(["--max-num-seqs", str(mocker_args["max_num_seqs"])])
    if "max_num_batched_tokens" in mocker_args:
        command.extend(
            ["--max-num-batched-tokens", str(mocker_args["max_num_batched_tokens"])]
        )
    if "enable_prefix_caching" in mocker_args:
        if mocker_args["enable_prefix_caching"]:
            command.append("--enable-prefix-caching")
        else:
            command.append("--no-enable-prefix-caching")
    if "enable_chunked_prefill" in mocker_args:
        if mocker_args["enable_chunked_prefill"]:
            command.append("--enable-chunked-prefill")
        else:
            command.append("--no-enable-chunked-prefill")
    if "watermark" in mocker_args:
        command.extend(["--watermark", str(mocker_args["watermark"])])
    if "dp_size" in mocker_args:
        command.extend(["--data-parallel-size", str(mocker_args["dp_size"])])
166
167
168
    # Use --durable-kv-events to enable JetStream mode (local indexer disabled)
    if mocker_args.get("durable_kv_events") is True:
        command.append("--durable-kv-events")
169
170
    if "bootstrap_ports" in mocker_args:
        command.extend(["--bootstrap-ports", mocker_args["bootstrap_ports"]])
171
172
    if "zmq_kv_events_ports" in mocker_args:
        command.extend(["--zmq-kv-events-ports", mocker_args["zmq_kv_events_ports"]])
173
174
175
176

    return command


177
class MockerProcess:
178
179
180
181
182
183
    """Manages mocker engine instances with shared tokio runtime via --num-workers.

    When standalone_indexer=True, launches mockers one-by-one (each as --num-workers 1)
    and runs a standalone HTTP KV indexer binary alongside them. Call launch_mockers_with_indexer()
    in async context to start mockers and register their ZMQ ports with the indexer.
    """
184

185
186
187
188
189
    def __init__(
        self,
        request,
        mocker_args: Optional[Dict[str, Any]] = None,
        num_mockers: int = 1,
190
        store_backend: str = "etcd",
191
        request_plane: str = "nats",
192
        zmq_kv_events: bool = False,
193
        standalone_indexer: bool = False,
194
        model_name: str = "mocker",
195
    ):
196
197
        namespace_suffix = generate_random_suffix()
        self.namespace = f"test-namespace-{namespace_suffix}"
198
        self.component_name = "mocker"
199
        self.model_name = model_name
200
        self.endpoint = f"dyn://{self.namespace}.{self.component_name}.generate"
201
        self.num_workers = num_mockers
202
        self._zmq_kv_events_ports: list[int] = []
203
204
205
206
207
208
209
210
211
212
213
        self._standalone_indexer = standalone_indexer
        self._standalone_indexer_port: Optional[int] = None
        self._indexer_process: Optional[ManagedProcess] = None
        self._mocker_processes: list[ManagedProcess] = []
        self._request = request
        self._store_backend = store_backend
        self._request_plane = request_plane
        self._mocker_args_orig: Dict[str, Any] = (mocker_args or {}).copy()
        self.worker_id_to_zmq_ports: dict[int, dict[int, str]] = {}

        mocker_args = self._mocker_args_orig.copy()
214
215
        # Store dp_size for DP-aware test functions
        self.dp_size = mocker_args.get("dp_size")
216
217
        # Alias for consistency with vLLM/SGLang workers
        self.data_parallel_size = self.dp_size
218

219
220
221
222
223
224
225
226
227
228
        # Allocate ZMQ base ports for KV event publishing.
        # Each worker's DP ranks bind on base_port + dp_rank, so we need bases
        # spaced dp_size apart. Allocate num_mockers * dp_size ports total,
        # then pick every dp_size'th port as a base.
        if zmq_kv_events:
            dp_size = mocker_args.get("dp_size", 1)
            self._zmq_kv_events_ports = allocate_ports(
                num_mockers * dp_size, BASE_PORT_ZMQ
            )
            bases = [self._zmq_kv_events_ports[i * dp_size] for i in range(num_mockers)]
229
230
            if not standalone_indexer:
                mocker_args["zmq_kv_events_ports"] = ",".join(str(p) for p in bases)
231
232
233
234
235
            logger.info(
                f"Allocated ZMQ KV event ports {self._zmq_kv_events_ports} "
                f"(bases: {bases}) for {num_mockers} workers"
            )

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
        if standalone_indexer:
            # Allocate a port for the standalone indexer HTTP server
            self._standalone_indexer_port = allocate_ports(1, BASE_PORT)[0]
            request.addfinalizer(
                lambda: deallocate_ports([self._standalone_indexer_port])
                if self._standalone_indexer_port
                else None
            )
            # Don't build a single mocker command — we'll launch per-mocker in launch_mockers_with_indexer
            self._process = None
        else:
            command = _build_mocker_command(
                endpoint=self.endpoint,
                store_backend=store_backend,
                num_workers=num_mockers,
                mocker_args=mocker_args,
            )
253

254
255
256
257
258
259
260
261
262
263
264
265
266
            env = os.environ.copy()
            env["DYN_REQUEST_PLANE"] = request_plane

            self._process = ManagedProcess(
                command=command,
                env=env,
                timeout=60,
                display_output=True,
                health_check_ports=[],
                health_check_urls=[],
                log_dir=request.node.name,
                terminate_all_matching_process_names=False,
            )
267
268
        logger.info(
            f"Created mocker process with {num_mockers} worker(s), endpoint: {self.endpoint}"
269
            f"{', standalone_indexer=True' if standalone_indexer else ''}"
270
271
        )

272
273
274
275
276
277
    @property
    def standalone_indexer_url(self) -> Optional[str]:
        if self._standalone_indexer_port is not None:
            return f"http://localhost:{self._standalone_indexer_port}"
        return None

278
    def __enter__(self):
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
        if self._standalone_indexer:
            # Launch the standalone indexer binary
            block_size = self._mocker_args_orig.get("block_size", BLOCK_SIZE)
            indexer_cmd = [
                "cargo",
                "run",
                "-p",
                "dynamo-kv-router",
                "--features",
                "indexer-bin",
                "--bin",
                "dynamo-kv-indexer",
                "--",
                "--block-size",
                str(block_size),
                "--port",
                str(self._standalone_indexer_port),
            ]
            self._indexer_process = ManagedProcess(
                command=indexer_cmd,
                timeout=120,
                display_output=True,
                health_check_ports=[self._standalone_indexer_port],
                health_check_urls=[],
                log_dir=self._request.node.name,
                terminate_all_matching_process_names=False,
                display_name="dynamo-kv-indexer",
            )
            logger.info(
                f"Starting standalone indexer on port {self._standalone_indexer_port}"
            )
            self._indexer_process.__enter__()
            # Don't start mocker processes yet — launch_mockers_with_indexer will do it
        else:
            logger.info(f"Starting mocker process with {self.num_workers} worker(s)")
            self._process.__enter__()
315
316
        return self

317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
    async def launch_mockers_with_indexer(self, endpoint):
        """Launch mockers one-by-one and register each with the standalone indexer.

        For each mocker:
        1. Launch a mocker process with --num-workers 1
        2. Poll endpoint.client().instance_ids() until a new worker_id appears
        3. POST /workers to the indexer with the worker_id and its ZMQ addresses

        Args:
            endpoint: The dynamo endpoint object to discover worker IDs.
        """
        client = await endpoint.client()
        known_ids: set[int] = set()
        dp_size = self._mocker_args_orig.get("dp_size", 1)

        for i in range(self.num_workers):
            # Build per-mocker args with its own ZMQ base port
            mocker_args = self._mocker_args_orig.copy()
            base_port = self._zmq_kv_events_ports[i * dp_size]
            mocker_args["zmq_kv_events_ports"] = str(base_port)

            command = _build_mocker_command(
                endpoint=self.endpoint,
                store_backend=self._store_backend,
                num_workers=1,
                mocker_args=mocker_args,
            )

            env = os.environ.copy()
            env["DYN_REQUEST_PLANE"] = self._request_plane

            proc = ManagedProcess(
                command=command,
                env=env,
                timeout=60,
                display_output=True,
                health_check_ports=[],
                health_check_urls=[],
                log_dir=self._request.node.name,
                terminate_all_matching_process_names=False,
                display_name=f"mocker-{i}",
            )
            proc.__enter__()
            self._mocker_processes.append(proc)

            # Poll for the new worker_id
            new_worker_id = None
            for _ in range(120):
                ids = set(client.instance_ids())
                new = ids - known_ids
                if new:
                    new_worker_id = new.pop()
                    known_ids.add(new_worker_id)
                    break
                await asyncio.sleep(0.5)

            if new_worker_id is None:
                raise RuntimeError(
                    f"Timed out waiting for mocker {i} to register "
                    f"(known_ids={known_ids})"
                )

            # Register each dp_rank endpoint with the standalone indexer
            zmq_addresses = {}
            register_url = f"{self.standalone_indexer_url}/register"
            async with aiohttp.ClientSession() as session:
                for dp_rank in range(dp_size):
                    port = self._zmq_kv_events_ports[i * dp_size + dp_rank]
                    endpoint = f"tcp://127.0.0.1:{port}"
                    zmq_addresses[str(dp_rank)] = endpoint

                    payload = {
                        "instance_id": new_worker_id,
                        "endpoint": endpoint,
                        "dp_rank": dp_rank,
392
393
394
395
                        "model_name": self.model_name,
                        "block_size": self._mocker_args_orig.get(
                            "block_size", BLOCK_SIZE
                        ),
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
                    }
                    async with session.post(register_url, json=payload) as resp:
                        if resp.status != 201:
                            body = await resp.text()
                            raise RuntimeError(
                                f"Failed to register instance {new_worker_id} "
                                f"dp_rank {dp_rank}: {resp.status} {body}"
                            )

            self.worker_id_to_zmq_ports[new_worker_id] = zmq_addresses

            logger.info(
                f"Mocker {i}: worker_id={new_worker_id}, "
                f"zmq_addresses={zmq_addresses}"
            )

        logger.info(
            f"All {self.num_workers} mockers launched and registered with indexer"
        )

416
    def __exit__(self, exc_type, exc_val, exc_tb):
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
        logger.info("Stopping mocker process(es)")
        # Stop individual mocker processes (standalone_indexer mode)
        for proc in self._mocker_processes:
            try:
                proc.__exit__(exc_type, exc_val, exc_tb)
            except Exception as e:
                logger.warning(f"Error stopping mocker process: {e}")
        self._mocker_processes.clear()
        # Stop standalone indexer
        if self._indexer_process is not None:
            try:
                self._indexer_process.__exit__(exc_type, exc_val, exc_tb)
            except Exception as e:
                logger.warning(f"Error stopping indexer process: {e}")
            self._indexer_process = None
        # Stop single mocker process (non-standalone mode)
        if self._process is not None:
            self._process.__exit__(exc_type, exc_val, exc_tb)
435
436
437
438
        if self._zmq_kv_events_ports:
            deallocate_ports(self._zmq_kv_events_ports)
            logger.info(f"Deallocated ZMQ KV event ports {self._zmq_kv_events_ports}")
            self._zmq_kv_events_ports = []
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458


class DisaggMockerProcess:
    """Manages prefill or decode mocker instances for disaggregated serving.

    Uses --num-workers for shared tokio runtime. For disaggregated serving:
    - Prefill workers: worker_type="prefill", endpoint is namespace.prefill.generate
    - Decode workers: worker_type="decode", endpoint is namespace.backend.generate

    Both prefill and decode workers should share the same namespace for proper discovery.
    """

    def __init__(
        self,
        request,
        namespace: str,
        worker_type: str,
        mocker_args: Optional[Dict[str, Any]] = None,
        num_mockers: int = 1,
        store_backend: str = "etcd",
459
        request_plane: str = "nats",
460
        enable_bootstrap: bool = False,
461
462
463
464
    ):
        if worker_type not in ("prefill", "decode"):
            raise ValueError(
                f"worker_type must be 'prefill' or 'decode', got {worker_type}"
465
            )
466
467
468
469

        self.namespace = namespace
        self.worker_type = worker_type
        self.num_workers = num_mockers
470
        self._bootstrap_ports: list[int] = []
471
472
473
474
475
476
477
478
479

        # Set component name and endpoint based on worker type
        if worker_type == "prefill":
            self.component_name = "prefill"
            self.endpoint = f"dyn://{self.namespace}.prefill.generate"
        else:
            self.component_name = "backend"
            self.endpoint = f"dyn://{self.namespace}.backend.generate"

480
481
482
483
        mocker_args = (mocker_args or {}).copy()

        # Allocate bootstrap ports for prefill workers if enabled (one per worker)
        if enable_bootstrap and worker_type == "prefill":
484
            self._bootstrap_ports = allocate_ports(num_mockers, BASE_PORT_BOOTSTRAP)
485
486
487
488
489
490
            mocker_args["bootstrap_ports"] = ",".join(
                str(p) for p in self._bootstrap_ports
            )
            logger.info(
                f"Allocated bootstrap ports {self._bootstrap_ports} for {num_mockers} prefill workers"
            )
491
492
493
494
495
496
497
498
499

        command = _build_mocker_command(
            endpoint=self.endpoint,
            store_backend=store_backend,
            num_workers=num_mockers,
            mocker_args=mocker_args,
            worker_type=worker_type,
        )

500
501
502
        env = os.environ.copy()
        env["DYN_REQUEST_PLANE"] = request_plane

503
504
        self._process = ManagedProcess(
            command=command,
505
            env=env,
506
507
508
509
510
            timeout=60,
            display_output=True,
            health_check_ports=[],
            health_check_urls=[],
            log_dir=request.node.name,
511
            terminate_all_matching_process_names=False,
512
513
514
515
516
        )
        logger.info(
            f"Created {worker_type} mocker process with {num_mockers} worker(s), "
            f"endpoint: {self.endpoint}"
        )
517

518
519
520
521
522
    @property
    def bootstrap_ports(self) -> list[int]:
        """Return the allocated bootstrap ports, if any."""
        return self._bootstrap_ports

523
    def __enter__(self):
524
525
526
527
        logger.info(
            f"Starting {self.worker_type} mocker process with {self.num_workers} worker(s)"
        )
        self._process.__enter__()
528
        return self
529

530
    def __exit__(self, exc_type, exc_val, exc_tb):
531
532
        logger.info(f"Stopping {self.worker_type} mocker process")
        self._process.__exit__(exc_type, exc_val, exc_tb)
533
534
535
536
537
        # Deallocate bootstrap ports if we allocated any
        if self._bootstrap_ports:
            deallocate_ports(self._bootstrap_ports)
            logger.info(f"Deallocated bootstrap ports {self._bootstrap_ports}")
            self._bootstrap_ports = []
538
539


540
@pytest.mark.timeout(120)  # bumped for xdist contention (was 42s; ~13.80s serial avg)
541
@pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True)
542
@pytest.mark.parametrize(
543
    "durable_kv_events", [False], indirect=True
544
)  # Use NATS Core (local indexer)
545
def test_mocker_kv_router(
546
547
548
549
    request,
    runtime_services_dynamic_ports,
    predownload_tokenizers,
    request_plane,
550
    durable_kv_events,
551
):
552
553
554
    """
    Test KV router with multiple mocker engine instances.
    This test doesn't require GPUs and runs quickly for pre-merge validation.
555
    Tests both NATS and TCP request planes.
556
557
    """

558
559
    # runtime_services starts etcd and optionally nats based on request_plane
    logger.info(f"Starting mocker KV router test with request_plane={request_plane}")
560

561
562
563
564
    # Create mocker args dictionary - use local indexer (NATS Core mode)
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
565
        "durable_kv_events": durable_kv_events,
566
    }
567

568
569
570
571
572
573
    with MockerProcess(
        request,
        mocker_args=mocker_args,
        num_mockers=NUM_MOCKERS,
        request_plane=request_plane,
    ) as mockers:
574
        # Start mocker instances with the new CLI interface
575
576
        logger.info(f"Starting {NUM_MOCKERS} mocker instances")
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")
577

578
        # Get unique port for this test
579
580
581
        frontend_port = get_unique_ports(
            request, num_ports=1, request_plane=request_plane
        )[0]
582
583

        # Run basic router test (starts router internally and waits for workers to be ready)
584
585
586
587
        _test_router_basic(
            engine_workers=mockers,
            block_size=BLOCK_SIZE,
            request=request,
588
            frontend_port=frontend_port,
589
590
            test_payload=TEST_PAYLOAD,
            num_requests=NUM_REQUESTS,
591
            request_plane=request_plane,
592
593
594
        )


595
@pytest.mark.parametrize("store_backend", ["etcd", "file"])
596
@pytest.mark.parametrize(
597
    "durable_kv_events", [False], indirect=True
598
)  # Use NATS Core (local indexer)
599
@pytest.mark.timeout(180)  # bumped for xdist contention (was 60s; ~19.86s serial avg)
600
601
def test_mocker_two_kv_router(
    request,
602
    runtime_services_dynamic_ports,
603
604
605
    predownload_tokenizers,
    file_storage_backend,
    store_backend,
606
    durable_kv_events,
607
):
608
609
610
    """
    Test with two KV routers and multiple mocker engine instances.
    Alternates requests between the two routers to test load distribution.
611
    Tests with both etcd and file storage backends.
612
613
614
    """

    # runtime_services starts etcd and nats
615
616
617
    logger.info(
        f"Starting mocker two KV router test with {store_backend} storage backend"
    )
618

619
620
621
622
    # Create mocker args dictionary - use local indexer (NATS Core mode)
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
623
        "durable_kv_events": durable_kv_events,
624
    }
625

626
627
628
629
630
631
    with MockerProcess(
        request,
        mocker_args=mocker_args,
        num_mockers=NUM_MOCKERS,
        store_backend=store_backend,
    ) as mockers:
632
        # Start mocker instances with the new CLI interface
633
634
        logger.info(f"Starting {NUM_MOCKERS} mocker instances")
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")
635

636
637
638
639
640
        # Get unique ports for this test (2 ports for two routers)
        router_ports = get_unique_ports(
            request, num_ports=2, store_backend=store_backend
        )

641
642
643
644
645
        # Run two-router test (starts KV routers internally and manages their lifecycle)
        _test_router_two_routers(
            engine_workers=mockers,
            block_size=BLOCK_SIZE,
            request=request,
646
            router_ports=router_ports,
647
648
649
            test_payload=TEST_PAYLOAD,
            num_requests=NUM_REQUESTS,
            store_backend=store_backend,
650
            skip_consumer_verification=not durable_kv_events,  # Skip JetStream checks in NATS Core mode
651
652
653
        )


654
@pytest.mark.skip(reason="Flaky, temporarily disabled")
655
@pytest.mark.parametrize(
656
    "durable_kv_events", [False], indirect=True
657
)  # Use NATS Core (local indexer)
658
@pytest.mark.timeout(60)  # ~3x average (~19.86s), rounded up (when enabled)
Alec's avatar
Alec committed
659
def test_mocker_kv_router_overload_503(
660
    request, runtime_services_dynamic_ports, predownload_tokenizers, durable_kv_events
Alec's avatar
Alec committed
661
):
662
    """Test that KV router returns 503 when mocker workers are overloaded."""
663
    logger.info("Starting mocker KV router overload test for 503 status")
664
    # Create mocker args dictionary with limited resources - use local indexer (NATS Core mode)
665
666
667
668
    mocker_args = {
        "speedup_ratio": 10,
        "block_size": 4,  # Smaller block size
        "num_gpu_blocks": 64,  # Limited GPU blocks to exhaust quickly
669
        "durable_kv_events": durable_kv_events,
670
    }
671

672
    with MockerProcess(request, mocker_args=mocker_args, num_mockers=1) as mockers:
673
        # Start single mocker instance with limited resources
674
675
        logger.info("Starting single mocker instance with limited resources")
        logger.info(f"Mocker using endpoint: {mockers.endpoint}")
676

677
678
679
        # Get unique port for this test
        frontend_port = get_unique_ports(request, num_ports=1)[0]

680
681
682
683
684
685
686
        # Run overload 503 test
        _test_router_overload_503(
            engine_workers=mockers,
            block_size=4,  # Match the mocker's block size
            request=request,
            frontend_port=frontend_port,
            test_payload=TEST_PAYLOAD,
687
            blocks_threshold=0.2,
688
        )
689

690

691
@pytest.mark.timeout(90)  # bumped for xdist contention (was 22s; ~7.10s serial avg)
692
@pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True)
693
@pytest.mark.parametrize(
694
    "durable_kv_events", [False], indirect=True
695
)  # Use NATS Core (local indexer)
696
def test_kv_router_bindings(
697
698
699
700
    request,
    runtime_services_dynamic_ports,
    predownload_tokenizers,
    request_plane,
701
    durable_kv_events,
702
):
703
704
    """Test KvRouter Python bindings with mocker engines."""
    logger.info("Starting KvRouter bindings test")
705
706
707
708
    # Use local indexer (NATS Core mode)
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
709
        "durable_kv_events": durable_kv_events,
710
    }
711

712
713
714
715
716
717
    with MockerProcess(
        request,
        mocker_args=mocker_args,
        num_mockers=NUM_MOCKERS,
        request_plane=request_plane,
    ) as mockers:
718
        # Start mocker instances
719
720
        logger.info(f"Starting {NUM_MOCKERS} mocker instances")
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")
721

722
        # Get runtime and create endpoint
723
        runtime = get_runtime(request_plane=request_plane)
724
725
726
        endpoint = runtime.endpoint(
            f"{mockers.namespace}.{mockers.component_name}.generate"
        )
727

728
729
730
        # Run Python router bindings test
        _test_python_router_bindings(
            engine_workers=mockers,
731
732
            endpoint=endpoint,
            block_size=BLOCK_SIZE,
733
734
            model_name=MODEL_NAME,
            num_workers=NUM_MOCKERS,
735
        )
736

737

738
@pytest.mark.parametrize(
739
    "store_backend,durable_kv_events,request_plane",
740
    [
741
742
743
        ("etcd", True, "nats"),  # JetStream mode - uses JetStream
        ("etcd", False, "tcp"),  # NATS core mode (with gap detection) - no JetStream
        ("file", True, "nats"),  # File backend - uses JetStream
744
    ],
745
746
    ids=[
        "jetstream",
747
        "nats_core",
748
        "file",
749
    ],
750
    indirect=["request_plane", "durable_kv_events"],
751
)
752
@pytest.mark.timeout(180)
753
754
def test_indexers_sync(
    request,
755
    runtime_services_dynamic_ports,
756
    predownload_tokenizers,
757
    build_kv_indexer,
758
759
    file_storage_backend,
    store_backend,
760
    durable_kv_events,
761
    request_plane,
762
):
763
764
765
766
    """
    Test that two KV routers have synchronized indexer states after processing requests.
    This test verifies that both routers converge to the same internal state.

767
768
    Tests with three configurations:
    - jetstream: etcd backend, JetStream for KV events, NATS request plane
769
    - nats_core: etcd backend, NATS Core with gap detection, TCP request plane
770
771
772
773
    - file: file backend, JetStream for KV events, NATS request plane
    """
    logger.info(
        f"Starting indexers sync test: store_backend={store_backend}, "
774
        f"durable_kv_events={durable_kv_events}, request_plane={request_plane}"
775
    )
776

777
778
779
780
    # Use the dynamic-port fixture to avoid hardcoded localhost:4222/2379 in parallel runs.
    nats_process, _etcd_process = runtime_services_dynamic_ports

    # Create mocker args dictionary
781
    # Use 2 DP ranks to test per-dp_rank event ID tracking and recovery
782
783
784
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
785
        "durable_kv_events": durable_kv_events,
786
        "dp_size": 2,
787
788
    }

789
790
791
792
793
794
795
    with MockerProcess(
        request,
        mocker_args=mocker_args,
        num_mockers=NUM_MOCKERS,
        store_backend=store_backend,
        request_plane=request_plane,
    ) as mockers:
796
797
        # Start mocker instances (2 workers x 2 DP ranks = 4 independent event streams)
        logger.info(f"Starting {NUM_MOCKERS} mocker instances with dp_size=2")
798
799
800
801
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")

        # Use the common test implementation (creates its own runtimes for each router)
        # Note: Consumer verification is done inside _test_router_indexers_sync while routers are alive
802
        # When using durable_kv_events=True, use JetStream mode for the router
803
804
805
806
807
808
809
        _test_router_indexers_sync(
            engine_workers=mockers,
            block_size=BLOCK_SIZE,
            model_name=MODEL_NAME,
            num_workers=NUM_MOCKERS,
            store_backend=store_backend,
            request_plane=request_plane,
810
811
812
            test_nats_interruption=not durable_kv_events,
            nats_server=nats_process if not durable_kv_events else None,
            durable_kv_events=durable_kv_events,
813
            standalone_indexer_url=mockers.standalone_indexer_url,
814
815
816
817
818
        )

        logger.info("Indexers sync test completed successfully")


819
@pytest.mark.timeout(120)  # bumped for xdist contention (was 42s; ~13.80s serial avg)
820
@pytest.mark.parametrize(
821
    "durable_kv_events", [False], indirect=True
822
)  # Use NATS Core (local indexer)
Alec's avatar
Alec committed
823
def test_query_instance_id_returns_worker_and_tokens(
824
    request, runtime_services_dynamic_ports, predownload_tokenizers, durable_kv_events
Alec's avatar
Alec committed
825
):
826
    """Test query_instance_id annotation with mocker engines."""
827
    logger.info("Starting KV router query_instance_id annotation test")
828
829
830
831
    # Use local indexer (NATS Core mode)
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
832
        "durable_kv_events": durable_kv_events,
833
    }
834

835
836
837
    with MockerProcess(
        request, mocker_args=mocker_args, num_mockers=NUM_MOCKERS
    ) as mockers:
838
        # Start mocker instances
839
840
        logger.info(f"Starting {NUM_MOCKERS} mocker instances")
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")
841

842
843
844
        # Get unique port for this test
        frontend_port = get_unique_ports(request, num_ports=1)[0]

845
846
847
848
849
850
851
852
        # Run query_instance_id annotation test
        _test_router_query_instance_id(
            engine_workers=mockers,
            block_size=BLOCK_SIZE,
            request=request,
            frontend_port=frontend_port,
            test_payload=TEST_PAYLOAD,
        )
853

854

855
@pytest.mark.timeout(90)  # bumped for xdist contention (was 29s; ~9.55s serial avg)
Yan Ru Pei's avatar
Yan Ru Pei committed
856
@pytest.mark.parametrize("request_plane", ["tcp"], indirect=True)
857
@pytest.mark.parametrize(
858
    "durable_kv_events,use_kv_events,zmq_kv_events",
859
    [
860
861
862
863
        (True, True, False),  # JetStream mode with KV events
        (False, True, False),  # NATS Core mode with local indexer (default)
        (False, False, False),  # Approximate mode (--no-kv-events) - no KV events
        (False, True, True),  # ZMQ mode: mocker → ZMQ PUB → relay → NATS
864
    ],
865
    ids=["jetstream", "nats_core", "no_kv_events", "zmq"],
866
    indirect=["durable_kv_events"],
867
)
868
def test_router_decisions(
869
870
871
    request,
    runtime_services_dynamic_ports,
    predownload_tokenizers,
872
    build_kv_indexer,
873
    durable_kv_events,
874
    use_kv_events,
875
    request_plane,
876
    zmq_kv_events,
877
878
879
):
    """Validate KV cache prefix reuse and dp_rank routing by sending progressive requests with overlapping prefixes.

880
    Parameterized to test:
881
882
    - JetStream mode: KV events via NATS JetStream (durable)
    - NATS Core mode (default): KV events via NATS Core with local indexer on workers
883
884
    - Approximate mode (--no-kv-events): No KV events, router predicts cache state
      based on routing decisions with TTL-based expiration and pruning
885
    """
886
    # runtime_services_dynamic_ports handles NATS and etcd startup
887
    logger.info(
888
        f"Starting test router decisions: durable_kv_events={durable_kv_events}, use_kv_events={use_kv_events}"
889
    )
890

Yan Ru Pei's avatar
Yan Ru Pei committed
891
    # Create mocker args dictionary with dp_size=4
892
    # durable_kv_events=True enables JetStream mode; False (default) uses NATS Core with local indexer
Yan Ru Pei's avatar
Yan Ru Pei committed
893
894
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
895
        "block_size": 8,
Yan Ru Pei's avatar
Yan Ru Pei committed
896
        "dp_size": 4,
897
        "durable_kv_events": durable_kv_events and use_kv_events,
Yan Ru Pei's avatar
Yan Ru Pei committed
898
    }
899

900
901
902
903
904
    with MockerProcess(
        request,
        mocker_args=mocker_args,
        num_mockers=2,
        request_plane=request_plane,
905
        zmq_kv_events=zmq_kv_events,
906
        standalone_indexer=zmq_kv_events,
907
        model_name=MODEL_NAME,
908
    ) as mockers:
909
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")
910

911
912
        # Initialize mockers
        # Get runtime and create endpoint
913
        runtime = get_runtime(request_plane=request_plane)
914
        # Use the namespace from the mockers
915
        endpoint = runtime.endpoint(f"{mockers.namespace}.mocker.generate")
916

917
        _test_router_decisions(
918
919
920
921
922
923
            mockers,
            endpoint,
            MODEL_NAME,
            request,
            test_dp_rank=True,
            use_kv_events=use_kv_events,
924
            durable_kv_events=durable_kv_events,
925
            standalone_indexer_url=mockers.standalone_indexer_url,
926
927
        )

928

929
@pytest.mark.parametrize("registration_order", ["prefill_first", "decode_first"])
930
931
932
@pytest.mark.parametrize(
    "enable_disagg_bootstrap", [False, True], ids=["no_bootstrap", "with_bootstrap"]
)
933
@pytest.mark.timeout(180)  # bumped for xdist contention (was 59s; ~19.51s serial avg)
934
def test_router_decisions_disagg(
935
936
937
938
    request,
    runtime_services_dynamic_ports,
    predownload_tokenizers,
    registration_order,
939
    enable_disagg_bootstrap,
940
941
942
943
944
):
    """Validate KV cache prefix reuse in disaggregated prefill-decode setup.

    Tests that progressive requests with overlapping prefixes are routed to the
    same prefill worker due to KV cache reuse.
945

946
947
948
    Parameterized to test:
    - registration_order: prefill_first vs decode_first
    - enable_disagg_bootstrap: without vs with bootstrap rendezvous
949
    """
950
    # runtime_services_dynamic_ports handles NATS and etcd startup
951
952
    logger.info(
        f"Starting disaggregated router prefix reuse test "
953
        f"(registration_order={registration_order}, bootstrap={enable_disagg_bootstrap})"
954
    )
955
956
957
958
959

    # Generate shared namespace for prefill and decode workers
    namespace_suffix = generate_random_suffix()
    shared_namespace = f"test-namespace-{namespace_suffix}"

960
    # Create mocker args - use NATS Core with local indexer (default mode)
961
962
963
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
964
        # durable_kv_events defaults to False (NATS Core mode)
965
    }
966

967
968
969
970
971
972
973
974
975
976
977
978
    if registration_order == "prefill_first":
        # Start prefill workers first
        logger.info("Starting 4 prefill mocker instances (first)")
        with DisaggMockerProcess(
            request,
            namespace=shared_namespace,
            worker_type="prefill",
            mocker_args=mocker_args,
            num_mockers=4,
            request_plane="nats",
            enable_bootstrap=enable_disagg_bootstrap,
        ) as prefill_workers:
979
980
981
982
            logger.info(f"Prefill workers using endpoint: {prefill_workers.endpoint}")

            # Then start decode workers
            logger.info("Starting 4 decode mocker instances (second)")
983
            with DisaggMockerProcess(
984
985
986
987
988
                request,
                namespace=shared_namespace,
                worker_type="decode",
                mocker_args=mocker_args,
                num_mockers=4,
989
                request_plane="nats",
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
            ) as decode_workers:
                logger.info(f"Decode workers using endpoint: {decode_workers.endpoint}")

                # Get unique port for this test
                frontend_port = get_unique_ports(
                    request, num_ports=1, registration_order=registration_order
                )[0]

                # Run disagg routing test
                _test_router_decisions_disagg(
                    prefill_workers=prefill_workers,
                    decode_workers=decode_workers,
                    block_size=BLOCK_SIZE,
                    request=request,
                    frontend_port=frontend_port,
                    test_payload=TEST_PAYLOAD,
                    request_plane="nats",
                )
    else:
        # Start decode workers first
        logger.info("Starting 4 decode mocker instances (first)")
        with DisaggMockerProcess(
            request,
            namespace=shared_namespace,
            worker_type="decode",
            mocker_args=mocker_args,
            num_mockers=4,
            request_plane="nats",
        ) as decode_workers:
1019
1020
1021
1022
            logger.info(f"Decode workers using endpoint: {decode_workers.endpoint}")

            # Then start prefill workers
            logger.info("Starting 4 prefill mocker instances (second)")
1023
            with DisaggMockerProcess(
1024
1025
1026
1027
1028
                request,
                namespace=shared_namespace,
                worker_type="prefill",
                mocker_args=mocker_args,
                num_mockers=4,
1029
1030
                request_plane="nats",
                enable_bootstrap=enable_disagg_bootstrap,
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
            ) as prefill_workers:
                logger.info(
                    f"Prefill workers using endpoint: {prefill_workers.endpoint}"
                )

                # Get unique port for this test
                frontend_port = get_unique_ports(
                    request, num_ports=1, registration_order=registration_order
                )[0]

                # Run disagg routing test
                _test_router_decisions_disagg(
                    prefill_workers=prefill_workers,
                    decode_workers=decode_workers,
                    block_size=BLOCK_SIZE,
                    request=request,
                    frontend_port=frontend_port,
                    test_payload=TEST_PAYLOAD,
                    request_plane="nats",
                )
1051
1052


1053
@pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True)
1054
@pytest.mark.parametrize(
1055
    "durable_kv_events", [False], indirect=True
1056
)  # Use NATS Core (local indexer)
1057
@pytest.mark.timeout(120)  # bumped for xdist contention (was 39s; ~12.84s serial avg)
1058
def test_busy_threshold_endpoint(
1059
1060
1061
1062
    request,
    runtime_services_dynamic_ports,
    predownload_tokenizers,
    request_plane,
1063
    durable_kv_events,
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
):
    """Test that the /busy_threshold endpoint can be hit and responds correctly.

    TODO: This doesn't actually test any e2e rejection for now. A proper test would:
    1. Set a very low threshold
    2. Send enough requests to exceed the threshold
    3. Verify that subsequent requests are rejected with 503

    For now, this test only verifies the endpoint is accessible and returns valid responses.
    """
1074
    # runtime_services_dynamic_ports handles NATS and etcd startup
1075
1076
1077
    logger.info(
        f"Starting busy_threshold endpoint test with request_plane={request_plane}"
    )
1078

1079
1080
1081
1082
    # Use local indexer (NATS Core mode)
    mocker_args = {
        "speedup_ratio": SPEEDUP_RATIO,
        "block_size": BLOCK_SIZE,
1083
        "durable_kv_events": durable_kv_events,
1084
    }
1085

1086
1087
1088
1089
1090
1091
    with MockerProcess(
        request,
        mocker_args=mocker_args,
        num_mockers=NUM_MOCKERS,
        request_plane=request_plane,
    ) as mockers:
1092
1093
1094
        logger.info(f"Starting {NUM_MOCKERS} mocker instances")
        logger.info(f"All mockers using endpoint: {mockers.endpoint}")

1095
1096
1097
        frontend_port = get_unique_ports(
            request, num_ports=1, request_plane=request_plane
        )[0]
1098
1099
1100
1101
1102
1103
1104

        _test_busy_threshold_endpoint(
            engine_workers=mockers,
            block_size=BLOCK_SIZE,
            request=request,
            frontend_port=frontend_port,
            test_payload=TEST_PAYLOAD,
1105
            request_plane=request_plane,
1106
        )