test: pytest mark checker (#4812)

Signed-off-by: pvijayakrish <pvijayakrish@nvidia.com>
Signed-off-by: Pavithra Vijayakrishnan <160681768+pvijayakrish@users.noreply.github.com>

.pre-commit-config.yaml

@@ -85,3 +85,16 @@ repos:
 # NOTE: pyright may be able to find other classes of errors not covered above,
 # but would require some configuring and venv setup to properly eliminate noise
 # and give it visiblity into all the local and third_party packages expected.
+- repo: local
+  hooks:
+    - id: pytest-marker-report
+      name: Report pytest markers (static + inherited)
+      entry: python3 scripts/report_pytest_markers.py
+      language: python
+      pass_filenames: false
+      additional_dependencies:
+        - pytest
+        - tomli
+        - pydantic
+        - filelock
+        - pyyaml

pyproject.toml

@@ -199,6 +199,7 @@ markers = [
     "parallel: marks tests that can run in parallel with pytest-xdist",
     "nightly: marks tests to run nightly",
     "weekly: marks tests to run weekly",
+    "release: marks tests to run on release pipelines",
     "gpu_0: marks tests that don't require GPU",
     "gpu_1: marks tests to run on GPU",
     "gpu_2: marks tests to run on 2GPUs",
@@ -209,6 +210,7 @@ markers = [
     "unit: marks tests as unit tests",
     "stress: marks tests as stress tests",
     "performance: marks tests as performance tests",
+    "benchmark: marks tests as benchmark tests",
     "vllm: marks tests as requiring vllm",
     "trtllm: marks tests as requiring trtllm",
     "sglang: marks tests as requiring sglang",
@@ -222,7 +224,15 @@ markers = [
     "model: model id used by a test or parameter",
     "custom_build: marks tests that require custom builds or special setup (e.g., MoE models)",
     "k8s: marks tests as requiring Kubernetes",
-    "fault_tolerance: marks tests as fault tolerance tests"
+    "fault_tolerance: marks tests as fault tolerance tests",
+    # Built-in markers
+    "skip: skip this test",
+    "skipif: skip if condition is true",
+    "xfail: expected failure",
+    "usefixtures: use fixtures",
+    "parametrize: parameterized test",
+    "filterwarnings: filter warnings",
+    "asyncio: asyncio test marker"
 ]
 
 # Linting/formatting

scripts/report_pytest_markers.py

+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Pytest Marker Report (Production Grade)
+
+- Collects pytest tests without executing them
+- Prints markers and validates category coverage
+- Optionally mocks unavailable dependencies so tests in import paths do
+  not fail collection
+- Provides structured output suitable for CI (text, JSON)
+"""
+
+from __future__ import annotations
+
+import argparse
+import configparser
+import importlib
+import json
+import logging
+import os
+import re
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from types import ModuleType
+from typing import Dict, List, Optional, Set
+from unittest.mock import MagicMock
+
+import pytest
+
+try:
+    import tomllib  # Python >=3.11
+except ImportError:
+    import tomli as tomllib  # type: ignore
+
+# --------------------------------------------------------------------------- #
+# Logging
+# --------------------------------------------------------------------------- #
+
+LOG = logging.getLogger("pytest-marker-report")
+# Disable all logging except CRITICAL to suppress noise from test code collection
+logging.disable(logging.WARNING)
+
+# --------------------------------------------------------------------------- #
+# Configuration
+# --------------------------------------------------------------------------- #
+
+REQUIRED_CATEGORIES: Dict[str, Set[str]] = {
+    "Lifecycle": {"pre_merge", "post_merge", "nightly", "weekly", "release"},
+    "Test Type": {
+        "unit",
+        "integration",
+        "e2e",
+        "benchmark",
+        "stress",
+        "multimodal",
+        "performance",
+    },
+    "Hardware": {"gpu_0", "gpu_1", "gpu_2", "gpu_4", "gpu_8", "h100", "k8s"},
+}
+
+STUB_MODULES = [
+    "pytest_httpserver",
+    "pytest_httpserver.HTTPServer",
+    "pytest_benchmark",
+    "pytest_benchmark.logger",
+    "pytest_benchmark.plugin",
+    "kubernetes",
+    "kubernetes_asyncio",
+    "kubernetes_asyncio.client",
+    "kubernetes_asyncio.client.exceptions",
+    "kubernetes.client",
+    "kubernetes.config",
+    "kubernetes.config.config_exception",
+    "kr8s",
+    "kr8s.objects",
+    "tritonclient",
+    "tritonclient.grpc",
+    "aiohttp",
+    "aiofiles",
+    "httpx",
+    "tabulate",
+    "prometheus_api_client",
+    "huggingface_hub",
+    "huggingface_hub.model_info",
+    "transformers",
+    "pandas",
+    "matplotlib",
+    "matplotlib.pyplot",
+    "pmdarima",
+    "prophet",
+    "scipy",
+    "scipy.interpolate",
+    "nats",
+    "dynamo._core",
+    "psutil",
+    "requests",
+    "numpy",
+    "gradio",
+    "aiconfigurator",
+    "aiconfigurator.webapp",
+    "aiconfigurator.webapp.components",
+    "aiconfigurator.webapp.components.profiling",
+    "boto3",
+    "botocore",
+    "botocore.client",
+    "botocore.exceptions",
+]
+
+# Project paths for local imports
+PROJECT_PATHS = [
+    os.getcwd(),
+    os.path.join(os.getcwd(), "components", "src"),
+    os.path.join(os.getcwd(), "lib", "bindings", "python", "src"),
+]
+sys.path[:0] = PROJECT_PATHS  # prepend to sys.path
+
+# --------------------------------------------------------------------------- #
+# Helpers
+# --------------------------------------------------------------------------- #
+
+
+def sanitize(s: str, max_len: int = 200) -> str:
+    """Safe, trimmed string for output."""
+    s = re.sub(r"[^\x20-\x7E\n\t]", "", str(s))
+    return s if len(s) <= max_len else s[: max_len - 3] + "..."
+
+
+def missing_categories(markers: Set[str]) -> List[str]:
+    """Return required categories missing in a test's markers."""
+    return [
+        cat for cat, allowed in REQUIRED_CATEGORIES.items() if not (markers & allowed)
+    ]
+
+
+# --------------------------------------------------------------------------- #
+# Dependency Stubbing
+# --------------------------------------------------------------------------- #
+
+
+class DependencyStubber:
+    """Stub unavailable modules to allow test collection without real dependencies."""
+
+    def __init__(self):
+        self.stubbed: Set[str] = set()
+
+    def _create_module_stub(self, name: str) -> MagicMock:
+        """Create a stub module with proper Python module attributes."""
+        stub = MagicMock()
+        stub.__path__ = []
+        stub.__name__ = name
+        stub.__loader__ = None
+        stub.__spec__ = None
+        stub.__package__ = name.rsplit(".", 1)[0] if "." in name else name
+        return stub
+
+    def ensure_available(self, module_name: str) -> ModuleType:
+        """Ensure a module is available, stubbing it if not installed."""
+        if module_name in sys.modules:
+            return sys.modules[module_name]
+
+        parts = module_name.split(".")
+        parent_stubbed = any(
+            ".".join(parts[:i]) in self.stubbed for i in range(1, len(parts))
+        )
+
+        if not parent_stubbed:
+            try:
+                return importlib.import_module(module_name)
+            except (ImportError, AttributeError):
+                pass
+
+        # Create parent packages if needed
+        for i in range(1, len(parts)):
+            sub = ".".join(parts[:i])
+            if sub not in sys.modules:
+                pkg = ModuleType(sub)
+                pkg.__path__ = []
+                sys.modules[sub] = pkg
+                self.stubbed.add(sub)
+
+        # Create stub module with proper attributes
+        stub = self._create_module_stub(module_name)
+        sys.modules[module_name] = stub
+        self.stubbed.add(module_name)
+        return stub
+
+
+# --------------------------------------------------------------------------- #
+# Data Structures
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class TestRecord:
+    nodeid: str
+    markers: List[str]
+    missing: List[str]
+
+
+@dataclass
+class Report:
+    total_checked: int
+    total_skipped_mypy: int
+    total_missing: int
+    tests: List[TestRecord]
+    undeclared_markers: Optional[List[str]] = None
+    missing_in_project_config: Optional[List[str]] = None
+
+
+# --------------------------------------------------------------------------- #
+# Pytest Plugin
+# --------------------------------------------------------------------------- #
+
+
+class MarkerReportPlugin:
+    def __init__(self):
+        self.records: List[TestRecord] = []
+        self.checked = 0
+        self.skipped_mypy = 0
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        for item in items:
+            markers = {m.name for m in item.iter_markers()}
+            if "mypy" in markers:
+                self.skipped_mypy += 1
+                continue
+
+            record = TestRecord(
+                nodeid=sanitize(item.nodeid),
+                markers=sorted(markers),
+                missing=missing_categories(markers),
+            )
+            self.records.append(record)
+            self.checked += 1
+
+    def build_report(self) -> Report:
+        return Report(
+            total_checked=self.checked,
+            total_skipped_mypy=self.skipped_mypy,
+            total_missing=sum(bool(r.missing) for r in self.records),
+            tests=self.records,
+        )
+
+
+# --------------------------------------------------------------------------- #
+# Marker Validation
+# --------------------------------------------------------------------------- #
+
+
+def load_declared_markers(project_root: Path = Path(".")) -> Set[str]:
+    """Load declared pytest markers from pytest.ini and pyproject.toml."""
+    declared: Set[str] = set()
+
+    # pytest.ini
+    ini_path = project_root / "pytest.ini"
+    if ini_path.exists():
+        cfg = configparser.ConfigParser()
+        cfg.read(str(ini_path))
+        markers = cfg.get("pytest", "markers", fallback="")
+        declared.update(
+            line.split(":", 1)[0].strip()
+            for line in markers.splitlines()
+            if line.strip()
+        )
+
+    # pyproject.toml
+    toml_path = project_root / "pyproject.toml"
+    if toml_path.exists():
+        try:
+            with toml_path.open("rb") as f:
+                data = tomllib.load(f)
+            markers_list = (
+                data.get("tool", {})
+                .get("pytest", {})
+                .get("ini_options", {})
+                .get("markers", [])
+            )
+            declared.update(
+                line.split(":", 1)[0].strip() for line in markers_list if line.strip()
+            )
+        except Exception as e:
+            LOG.warning("Failed reading pyproject.toml markers: %s", e)
+
+    return declared
+
+
+def validate_marker_definitions(report: Report, declared: Set[str]) -> None:
+    """Fill report with metadata about declared/undeclared markers."""
+    used = {m for t in report.tests for m in t.markers}
+    required = {m for s in REQUIRED_CATEGORIES.values() for m in s}
+
+    report.undeclared_markers = sorted(used - declared) or None
+    report.missing_in_project_config = sorted(required - declared) or None
+
+
+class MarkerStrictValidator:
+    """Strict validation for marker definitions and naming conventions."""
+
+    NAME_PATTERN = re.compile(r"^[a-z0-9_]+$")
+
+    @staticmethod
+    def validate(report: Report, declared: Set[str]) -> List[str]:
+        """Return list of validation errors (empty if valid)."""
+        errors: List[str] = []
+
+        if report.undeclared_markers:
+            errors.append(
+                "Undeclared markers used: " + ", ".join(report.undeclared_markers)
+            )
+
+        if report.missing_in_project_config:
+            errors.append(
+                "Required markers missing in pytest.ini/pyproject.toml: "
+                + ", ".join(report.missing_in_project_config)
+            )
+
+        bad_names = sorted(
+            m for m in declared if not MarkerStrictValidator.NAME_PATTERN.fullmatch(m)
+        )
+        if bad_names:
+            errors.append(
+                "Invalid marker names (must match [a-z0-9_]+): " + ", ".join(bad_names)
+            )
+
+        return errors
+
+
+# --------------------------------------------------------------------------- #
+# CLI & Runner
+# --------------------------------------------------------------------------- #
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="pytest marker validator")
+    parser.add_argument("--json", help="Write JSON report to file")
+    parser.add_argument(
+        "--no-stub", action="store_true", help="Disable dependency stubbing"
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Enable strict validation (undeclared markers, missing config, naming)",
+    )
+    parser.add_argument(
+        "--tests", default="tests", help="Path to test directory (default: tests)"
+    )
+    return parser.parse_args()
+
+
+def run_collection(test_path: str, use_stubbing: bool) -> tuple[int, Report]:
+    """Run pytest collection and return exit code and report."""
+    if use_stubbing:
+        stubber = DependencyStubber()
+        for module in STUB_MODULES:
+            stubber.ensure_available(module)
+
+        # Special case: pytest-benchmark needs a real Warning subclass
+        try:
+            sys.modules["pytest_benchmark.logger"].PytestBenchmarkWarning = type(  # type: ignore[attr-defined]
+                "PytestBenchmarkWarning", (Warning,), {}
+            )
+        except (KeyError, AttributeError):
+            pass
+
+        LOG.info("Stubbed %d modules", len(stubber.stubbed))
+
+    plugin = MarkerReportPlugin()
+    exitcode = pytest.main(
+        [
+            "--collect-only",
+            "-qq",
+            "--disable-warnings",
+            # Override config from pyproject.toml to avoid picking up options
+            # that require plugins/modules not installed in this environment
+            "-o",
+            "addopts=",
+            "-o",
+            "filterwarnings=",
+            test_path,
+        ],
+        plugins=[plugin],
+    )
+    return exitcode, plugin.build_report()
+
+
+def print_human_report(report: Report) -> None:
+    """Print human-readable report to stdout."""
+    print("\n" + "=" * 80)
+    print(f"{'TEST ID':<60} | MARKERS")
+    print("=" * 80)
+
+    for rec in report.tests:
+        print(f"{rec.nodeid:<60} | {', '.join(rec.markers)}")
+
+    # Print tests with missing markers before summary
+    missing_tests = [rec for rec in report.tests if rec.missing]
+    if missing_tests:
+        print("\n" + "=" * 80)
+        print("TESTS MISSING REQUIRED MARKERS")
+        print("=" * 80)
+        for rec in missing_tests:
+            print(f"{rec.nodeid}")
+            print(f"  Missing: {', '.join(rec.missing)}")
+
+    print("\n" + "=" * 80)
+    print("SUMMARY")
+    print("=" * 80)
+    print(f"  Tests checked: {report.total_checked}")
+    print(f"  Mypy skipped:  {report.total_skipped_mypy}")
+    print(f"  Missing sets:  {report.total_missing}")
+    print("=" * 80)
+
+
+def main() -> int:
+    """Main entry point."""
+    args = parse_args()
+    exitcode, report = run_collection(args.tests, not args.no_stub)
+
+    # Load and validate marker definitions
+    declared = load_declared_markers(Path("."))
+    validate_marker_definitions(report, declared)
+
+    print_human_report(report)
+
+    # Strict mode validation
+    if args.strict:
+        strict_errors = MarkerStrictValidator.validate(report, declared)
+        if strict_errors:
+            for e in strict_errors:
+                LOG.error("[STRICT] %s", e)
+            return 1
+
+    # Write JSON report if requested
+    if args.json:
+        with open(args.json, "w", encoding="utf-8") as f:
+            json.dump(asdict(report), f, indent=2)
+        LOG.info("Wrote JSON report to %s", args.json)
+
+    # Fail if any tests are missing required markers
+    return 1 if report.total_missing > 0 else exitcode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/basic/test_autodeploy_backend.py

@@ -169,6 +169,7 @@ def send_completion_request(
 @pytest.mark.e2e
 @pytest.mark.slow
 @pytest.mark.gpu_1
+@pytest.mark.nightly
 def test_smoke(request, runtime_services):
     """End-to-end test for TRTLLM worker with autodeploy backend in its most basic form."""
 

tests/ci/test_build_sh.py

@@ -23,7 +23,7 @@ from pathlib import Path
 
 import pytest
 
-pytestmark = [pytest.mark.unit, pytest.mark.pre_merge]
+pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0]
 
 
 @pytest.fixture(scope="module")

tests/dependencies/test_kvbm_imports.py

@@ -43,12 +43,16 @@ def _check_kvbm_imports():
 
 # Base tests (no framework markers) - run in main job with --framework none --enable-kvbm
 @pytest.mark.pre_merge
+@pytest.mark.gpu_0
+@pytest.mark.unit
 def test_kvbm_wheel_exists():
     """Verify KVBM wheel file exists in expected location."""
     _check_kvbm_wheel_exists()
 
 
 @pytest.mark.pre_merge
+@pytest.mark.gpu_0
+@pytest.mark.unit
 def test_kvbm_imports():
     """Verify KVBM package and core classes can be imported."""
     _check_kvbm_imports()
@@ -57,6 +61,8 @@ def test_kvbm_imports():
 # vLLM-specific tests - run in vLLM job (vLLM auto-enables KVBM)
 @pytest.mark.pre_merge
 @pytest.mark.vllm
+@pytest.mark.unit
+@pytest.mark.gpu_0
 def test_kvbm_wheel_exists_vllm():
     """Verify KVBM wheel exists in vLLM image."""
     _check_kvbm_wheel_exists()
@@ -64,6 +70,8 @@ def test_kvbm_wheel_exists_vllm():
 
 @pytest.mark.pre_merge
 @pytest.mark.vllm
+@pytest.mark.unit
+@pytest.mark.gpu_0
 def test_kvbm_imports_vllm():
     """Verify KVBM package and core classes can be imported in vLLM image."""
     _check_kvbm_imports()
@@ -72,6 +80,8 @@ def test_kvbm_imports_vllm():
 # TRT-LLM-specific tests - run in TRT-LLM job (TRT-LLM auto-enables KVBM)
 @pytest.mark.pre_merge
 @pytest.mark.trtllm
+@pytest.mark.unit
+@pytest.mark.gpu_0
 def test_kvbm_wheel_exists_trtllm():
     """Verify KVBM wheel exists in TRT-LLM image."""
     _check_kvbm_wheel_exists()
@@ -79,6 +89,8 @@ def test_kvbm_wheel_exists_trtllm():
 
 @pytest.mark.pre_merge
 @pytest.mark.trtllm
+@pytest.mark.unit
+@pytest.mark.gpu_0
 def test_kvbm_imports_trtllm():
     """Verify KVBM package and core classes can be imported in TRT-LLM image."""
     _check_kvbm_imports()

tests/dependencies/test_vllm_imports.py

@@ -9,6 +9,7 @@ import pytest
 @pytest.mark.vllm
 @pytest.mark.unit
 @pytest.mark.gpu_1
+@pytest.mark.pre_merge
 def test_import_deep_ep():
     """Test that deep_ep module can be imported."""
     try:
@@ -22,6 +23,7 @@ def test_import_deep_ep():
 @pytest.mark.vllm
 @pytest.mark.unit
 @pytest.mark.gpu_1
+@pytest.mark.pre_merge
 def test_import_pplx_kernels():
     """Test that pplx_kernels module can be imported."""
     try:

tests/fault_tolerance/cancellation/test_trtllm.py

@@ -34,6 +34,7 @@ pytestmark = [
     pytest.mark.trtllm,
     pytest.mark.gpu_1,
     pytest.mark.e2e,
+    pytest.mark.post_merge,
     pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME),
     pytest.mark.post_merge,  # post_merge to pinpoint failure commit
     pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True),

tests/fault_tolerance/deploy/test_deployment.py

@@ -435,6 +435,7 @@ def results_summary():
 
 @pytest.mark.k8s
 @pytest.mark.fault_tolerance
+@pytest.mark.post_merge
 @pytest.mark.e2e
 @pytest.mark.slow
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")

tests/fault_tolerance/etcd_ha/test_sglang.py

@@ -151,6 +151,7 @@ class DynamoWorkerProcess(ManagedProcess):
 @pytest.mark.sglang
 @pytest.mark.gpu_1
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_ha_failover_sglang_aggregated(request, predownload_models):
@@ -224,6 +225,7 @@ def test_etcd_ha_failover_sglang_aggregated(request, predownload_models):
 @pytest.mark.sglang
 @pytest.mark.gpu_2
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_ha_failover_sglang_disaggregated(
@@ -305,6 +307,7 @@ def test_etcd_ha_failover_sglang_disaggregated(
 @pytest.mark.sglang
 @pytest.mark.gpu_1
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_non_ha_shutdown_sglang_aggregated(request, predownload_models):
@@ -362,6 +365,7 @@ def test_etcd_non_ha_shutdown_sglang_aggregated(request, predownload_models):
 @pytest.mark.sglang
 @pytest.mark.gpu_2
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_non_ha_shutdown_sglang_disaggregated(

tests/fault_tolerance/etcd_ha/test_trtllm.py

@@ -133,6 +133,7 @@ class DynamoWorkerProcess(ManagedProcess):
 @pytest.mark.trtllm
 @pytest.mark.gpu_1
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_ha_failover_trtllm_aggregated(request, predownload_models):
@@ -206,6 +207,7 @@ def test_etcd_ha_failover_trtllm_aggregated(request, predownload_models):
 @pytest.mark.trtllm
 @pytest.mark.gpu_1
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_ha_failover_trtllm_disaggregated(
@@ -286,6 +288,7 @@ def test_etcd_ha_failover_trtllm_disaggregated(
 @pytest.mark.trtllm
 @pytest.mark.gpu_1
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_non_ha_shutdown_trtllm_aggregated(request, predownload_models):
@@ -346,6 +349,7 @@ def test_etcd_non_ha_shutdown_trtllm_aggregated(request, predownload_models):
 @pytest.mark.trtllm
 @pytest.mark.gpu_1
 @pytest.mark.e2e
+@pytest.mark.nightly
 @pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME)
 @pytest.mark.timeout(600)
 def test_etcd_non_ha_shutdown_trtllm_disaggregated(

tests/kvbm_integration/test_consolidator_router_e2e.py

@@ -57,6 +57,7 @@ pytestmark = [
     pytest.mark.e2e,
     pytest.mark.slow,
     pytest.mark.gpu_1,
+    pytest.mark.pre_merge,
     pytest.mark.skipif(not (HAS_VLLM or HAS_TRTLLM), reason="requires vllm or trtllm"),
 ]
 

tests/profiler/test_profile_sla_aiconfigurator.py

@@ -114,7 +114,7 @@ class TestProfileSlaAiconfigurator:
     @pytest.mark.parallel
     @pytest.mark.asyncio
     @pytest.mark.gpu_1
-    @pytest.mark.performance
+    @pytest.mark.integration
     async def test_trtllm_aiconfigurator_single_model(self, llm_args):
         # Test that profile_sla works with the model & backend in the llm_args fixture.
         await run_profile(llm_args)
@@ -122,8 +122,8 @@ class TestProfileSlaAiconfigurator:
     @pytest.mark.parallel
     @pytest.mark.asyncio
     @pytest.mark.gpu_1
+    @pytest.mark.integration
     @pytest.mark.nightly
-    @pytest.mark.performance
     @pytest.mark.parametrize(
         "backend, aic_backend_version",
         [

tests/router/test_router_e2e_with_sglang.py

@@ -413,6 +413,7 @@ def test_router_decisions_sglang_multiple_workers(
 
 
 @pytest.mark.gpu_2
+@pytest.mark.post_merge
 @pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True)
 @pytest.mark.timeout(600)  # 10 min max (multi-GPU + DP startup variance)
 def test_router_decisions_sglang_dp(

tests/router/test_router_e2e_with_vllm.py

@@ -427,6 +427,7 @@ def test_router_decisions_vllm_multiple_workers(
 
 
 @pytest.mark.gpu_2
+@pytest.mark.nightly
 @pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True)
 @pytest.mark.timeout(600)  # 10 min max (multi-GPU + DP startup variance)
 def test_router_decisions_vllm_dp(

tests/serve/test_sglang.py

@@ -114,7 +114,7 @@ sglang_configs = {
         name="kv_events",
         directory=sglang_dir,
         script_name="agg_router.sh",
-        marks=[pytest.mark.gpu_2],
+        marks=[pytest.mark.gpu_2, pytest.mark.post_merge],
         model="Qwen/Qwen3-0.6B",
         env={
             "DYN_LOG": "dynamo_llm::kv_router::publisher=trace,dynamo_llm::kv_router::scheduler=info",
@@ -227,6 +227,7 @@ sglang_configs = {
         script_name="agg.sh",
         marks=[
             pytest.mark.gpu_1,
+            pytest.mark.post_merge,
             pytest.mark.timeout(
                 420
             ),  # Total test timeout: 2x measured average (79.36s) + download time (240s) for 7B model
@@ -276,7 +277,7 @@ def test_sglang_deployment(
 
 @pytest.mark.e2e
 @pytest.mark.sglang
-@pytest.mark.gpu_1
+@pytest.mark.gpu_2
 @pytest.mark.nightly
 @pytest.mark.skip(
     reason="Requires 4 GPUs - enable when hardware is consistently available"

tests/serve/test_trtllm.py

@@ -171,7 +171,12 @@ trtllm_configs = {
         name="disaggregated_multimodal",
         directory=trtllm_dir,
         script_name="disagg_multimodal.sh",
-        marks=[pytest.mark.gpu_2, pytest.mark.trtllm, pytest.mark.multimodal],
+        marks=[
+            pytest.mark.gpu_2,
+            pytest.mark.trtllm,
+            pytest.mark.multimodal,
+            pytest.mark.nightly,
+        ],
         model="Qwen/Qwen2-VL-7B-Instruct",
         frontend_port=DefaultPort.FRONTEND.value,
         timeout=900,
@@ -185,6 +190,7 @@ trtllm_configs = {
         marks=[
             pytest.mark.gpu_1,
             pytest.mark.trtllm,
+            pytest.mark.post_merge,
             pytest.mark.timeout(
                 480
             ),  # 3x measured time (83.85s) + download time (210s) for 7B model
@@ -244,6 +250,7 @@ def test_deployment(
 @pytest.mark.e2e
 @pytest.mark.gpu_1
 @pytest.mark.trtllm
+@pytest.mark.pre_merge
 @pytest.mark.timeout(660)  # 3x measured time (159.68s) + download time (180s)
 def test_chat_only_aggregated_with_test_logits_processor(
     request,

tests/serve/test_vllm.py

@@ -83,7 +83,7 @@ vllm_configs = {
         name="aggregated_logprobs",
         directory=vllm_dir,
         script_name="agg.sh",
-        marks=[pytest.mark.gpu_1],
+        marks=[pytest.mark.gpu_1, pytest.mark.post_merge],
         model="Qwen/Qwen3-0.6B",
         request_payloads=[
             chat_payload_with_logprobs(
@@ -125,6 +125,7 @@ vllm_configs = {
         script_name="agg_lmcache_multiproc.sh",
         marks=[
             pytest.mark.gpu_1,
+            pytest.mark.pre_merge,
             pytest.mark.timeout(360),  # 3x estimated time (70s) + download time (150s)
         ],
         model="Qwen/Qwen3-0.6B",
@@ -339,6 +340,7 @@ vllm_configs = {
         script_name="agg_multimodal.sh",
         marks=[
             pytest.mark.gpu_1,
+            pytest.mark.nightly,
             # https://github.com/ai-dynamo/dynamo/issues/4501
             pytest.mark.xfail(strict=False),
         ],
@@ -428,7 +430,7 @@ vllm_configs = {
         name="aggregated_toolcalling",
         directory=vllm_dir,
         script_name="agg_multimodal.sh",
-        marks=[pytest.mark.gpu_2, pytest.mark.multimodal],
+        marks=[pytest.mark.gpu_2, pytest.mark.multimodal, pytest.mark.nightly],
         model="Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
         script_args=[
             "--model",
@@ -508,6 +510,7 @@ vllm_configs = {
         script_name="agg.sh",
         marks=[
             pytest.mark.gpu_1,
+            pytest.mark.post_merge,
             pytest.mark.timeout(
                 420
             ),  # 3x estimated time (60s) + download time (240s) for 7B model
@@ -594,7 +597,6 @@ def vllm_config_test(request):
 
 @pytest.mark.vllm
 @pytest.mark.e2e
-@pytest.mark.nightly
 def test_serve_deployment(
     vllm_config_test,
     request,
@@ -615,6 +617,7 @@ def test_serve_deployment(
 @pytest.mark.vllm
 @pytest.mark.e2e
 @pytest.mark.gpu_2
+@pytest.mark.nightly
 @pytest.mark.timeout(360)  # Match VLLMConfig.timeout for this multimodal deployment
 def test_multimodal_b64(
     request,