test: relocate test output to /tmp to keep git working tree clean (#6289)

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

tests/conftest.py

@@ -20,6 +20,7 @@ from tests.utils.port_utils import (
     deallocate_port,
     deallocate_ports,
 )
+from tests.utils.test_output import resolve_test_output_path
 
 _logger = logging.getLogger(__name__)
 
@@ -233,10 +234,11 @@ def predownload_tokenizers(pytestconfig):
 
 @pytest.fixture(autouse=True)
 def logger(request):
-    log_path = os.path.join(request.node.name, "test.log.txt")
+    log_dir = resolve_test_output_path(request.node.name)
+    log_path = os.path.join(log_dir, "test.log.txt")
     logger = logging.getLogger()
-    shutil.rmtree(request.node.name, ignore_errors=True)
-    os.makedirs(request.node.name, exist_ok=True)
+    shutil.rmtree(log_dir, ignore_errors=True)
+    os.makedirs(log_dir, exist_ok=True)
     handler = logging.FileHandler(log_path, mode="w")
     formatter = logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)
     handler.setFormatter(formatter)

tests/fault_tolerance/deploy/test_deployment.py

@@ -26,6 +26,7 @@ from tests.fault_tolerance.deploy.scenarios import (
     scenarios,
 )
 from tests.utils.managed_deployment import DeploymentSpec, ManagedDeployment
+from tests.utils.test_output import resolve_test_output_path
 
 
 @pytest.fixture
@@ -271,8 +272,8 @@ def validation_context(request, scenario):  # noqa: F811
 
     if hasattr(scenario.load, "mixed_token_test") and scenario.load.mixed_token_test:
         # For mixed token tests, we have separate overflow and recovery directories
-        overflow_dir = f"{request.node.name}{OVERFLOW_SUFFIX}"
-        recovery_dir = f"{request.node.name}{RECOVERY_SUFFIX}"
+        overflow_dir = resolve_test_output_path(f"{request.node.name}{OVERFLOW_SUFFIX}")
+        recovery_dir = resolve_test_output_path(f"{request.node.name}{RECOVERY_SUFFIX}")
         log_paths = [overflow_dir, recovery_dir]
 
         logging.info("Mixed token test detected. Looking for results in:")
@@ -280,7 +281,7 @@ def validation_context(request, scenario):  # noqa: F811
         logging.info(f"  - Recovery phase: {recovery_dir}")
     else:
         # Standard test with single directory
-        log_paths = [request.node.name]
+        log_paths = [resolve_test_output_path(request.node.name)]
 
     # Use factory to auto-detect and parse results
     try:
@@ -321,7 +322,7 @@ def validation_context(request, scenario):  # noqa: F811
                     # Create ValidationContext for all checkers
                     validation_ctx = ValidationContext(
                         scenario=scenario,
-                        log_dir=test_name,
+                        log_dir=resolve_test_output_path(test_name),
                         metrics=metrics,
                         deployment=context.get("deployment"),
                         namespace=context.get("namespace"),

tests/fault_tolerance/etcd_ha/utils.py

@@ -19,6 +19,7 @@ from tests.utils.managed_process import (
     DynamoFrontendProcess as BaseDynamoFrontendProcess,
 )
 from tests.utils.managed_process import ManagedProcess
+from tests.utils.test_output import resolve_test_output_path
 
 logger = logging.getLogger(__name__)
 
@@ -143,7 +144,9 @@ class EtcdCluster:
         self.base_port = base_port
         self.replicas: List[Optional[EtcdReplicaServer]] = []
         self.data_dirs: List[str] = []
-        self.log_base_dir = f"{request.node.name}_etcd_cluster"
+        self.log_base_dir = resolve_test_output_path(
+            f"{request.node.name}_etcd_cluster"
+        )
 
         # Clean up any existing log directory
         try:

tests/kvbm_integration/test_consolidator_router_e2e.py

@@ -25,6 +25,7 @@ import yaml
 
 from tests.kvbm_integration.common import ApiTester, check_logs_for_patterns
 from tests.utils.managed_process import ManagedProcess
+from tests.utils.test_output import resolve_test_output_path
 
 # Check if engines are available and build list of available engines
 from .common import check_module_available
@@ -58,7 +59,7 @@ FRONTEND_PORT = 8000
 @pytest.fixture
 def test_directory(request):
     """Create a test directory for logs and temporary files."""
-    test_dir = Path(request.node.name)
+    test_dir = Path(resolve_test_output_path(request.node.name))
     test_dir.mkdir(parents=True, exist_ok=True)
     yield test_dir
     # Cleanup handled by pytest (logs are kept for debugging)

tests/kvbm_integration/test_determinism_agg.py

@@ -35,6 +35,7 @@ import pytest
 import requests
 
 from tests.utils.port_utils import allocate_port, deallocate_port
+from tests.utils.test_output import resolve_test_output_path
 
 from .common import DeterminismTester, ServerType
 from .common import TestDeterminism as BaseTestDeterminism
@@ -404,7 +405,7 @@ def llm_server(request, runtime_services):
     port = getattr(request, "param", {}).get("port", None)
 
     # Put logs in the per-test directory set up by tests/conftest.py
-    log_dir = Path(request.node.name)
+    log_dir = Path(resolve_test_output_path(request.node.name))
 
     if check_module_available("vllm"):
         server_type = ServerType.vllm

tests/kvbm_integration/test_determinism_disagg.py

@@ -29,6 +29,8 @@ import pytest
 import requests
 import yaml
 
+from tests.utils.test_output import resolve_test_output_path
+
 from .common import DeterminismTester, ServerType
 from .common import TestDeterminism as BaseTestDeterminism
 from .common import check_module_available
@@ -507,7 +509,7 @@ def llm_server(request, runtime_services):
     port = getattr(request, "param", {}).get("port", None)
 
     # Put logs in the per-test directory set up by tests/conftest.py
-    log_dir = Path(request.node.name)
+    log_dir = Path(resolve_test_output_path(request.node.name))
 
     if check_module_available("vllm"):
         server_type = ServerType.vllm

tests/router/test_router_e2e_with_mockers.py

@@ -620,7 +620,6 @@ def test_query_instance_id_returns_worker_and_tokens(
         "block_size": BLOCK_SIZE,
         "durable_kv_events": durable_kv_events,
     }
-    os.makedirs(request.node.name, exist_ok=True)
 
     with MockerProcess(
         request, mocker_args=mocker_args, num_mockers=NUM_MOCKERS

tests/router/test_router_e2e_with_sglang.py

@@ -22,6 +22,7 @@ from tests.router.common import (  # utilities
 from tests.utils.constants import DefaultPort
 from tests.utils.managed_process import ManagedProcess
 from tests.utils.port_utils import allocate_ports, deallocate_ports
+from tests.utils.test_output import resolve_test_output_path
 
 logger = logging.getLogger(__name__)
 
@@ -261,6 +262,7 @@ class SGLangProcess:
                 # Manually initialize the process without blocking on health checks
                 process._logger = logging.getLogger(process.__class__.__name__)
                 process._command_name = process.command[0]
+                process.log_dir = resolve_test_output_path(process.log_dir)
                 os.makedirs(process.log_dir, exist_ok=True)
                 log_name = f"{process._command_name}.log.txt"
                 process._log_path = os.path.join(process.log_dir, log_name)

tests/router/test_router_e2e_with_trtllm.py

@@ -22,6 +22,7 @@ from tests.router.common import (  # utilities
 from tests.utils.constants import DefaultPort
 from tests.utils.managed_process import ManagedProcess
 from tests.utils.port_utils import allocate_ports, deallocate_ports
+from tests.utils.test_output import resolve_test_output_path
 
 logger = logging.getLogger(__name__)
 
@@ -242,6 +243,7 @@ class TRTLLMProcess:
                 # Manually initialize the process without blocking on health checks
                 process._logger = logging.getLogger(process.__class__.__name__)
                 process._command_name = process.command[0]
+                process.log_dir = resolve_test_output_path(process.log_dir)
                 os.makedirs(process.log_dir, exist_ok=True)
                 log_name = f"{process._command_name}.log.txt"
                 process._log_path = os.path.join(process.log_dir, log_name)

tests/router/test_router_e2e_with_vllm.py

@@ -22,6 +22,7 @@ from tests.router.common import (  # utilities
 from tests.utils.constants import DefaultPort
 from tests.utils.managed_process import ManagedProcess
 from tests.utils.port_utils import allocate_ports, deallocate_ports
+from tests.utils.test_output import resolve_test_output_path
 
 logger = logging.getLogger(__name__)
 
@@ -279,6 +280,7 @@ class VLLMProcess:
                 # Manually initialize the process without blocking on health checks
                 process._logger = logging.getLogger(process.__class__.__name__)
                 process._command_name = process.command[0]
+                process.log_dir = resolve_test_output_path(process.log_dir)
                 os.makedirs(process.log_dir, exist_ok=True)
                 log_name = f"{process._command_name}.log.txt"
                 process._log_path = os.path.join(process.log_dir, log_name)

tests/utils/managed_deployment.py

@@ -18,6 +18,8 @@ from kr8s.objects import Pod, Service
 from kubernetes_asyncio import client, config
 from kubernetes_asyncio.client import exceptions
 
+from tests.utils.test_output import resolve_test_output_path
+
 
 def _get_workspace_dir() -> str:
     """Get workspace directory without depending on dynamo.common package.
@@ -508,6 +510,7 @@ class ManagedDeployment:
 
     def __post_init__(self):
         self._deployment_name = self.deployment_spec.name
+        self.log_dir = resolve_test_output_path(self.log_dir)
 
     async def _init_kubernetes(self):
         """Initialize kubernetes client.

tests/utils/managed_process.py

@@ -8,7 +8,6 @@ import shutil
 import signal
 import socket
 import subprocess
-import tempfile
 import time
 from dataclasses import dataclass, field
 from typing import Any, List, Optional
@@ -18,6 +17,7 @@ import requests
 
 from tests.utils.constants import DefaultPort
 from tests.utils.port_utils import allocate_port, deallocate_port
+from tests.utils.test_output import resolve_test_output_path
 
 
 def terminate_process(process, logger=logging.getLogger(), immediate_kill=False):
@@ -182,12 +182,7 @@ class ManagedProcess:
             # Keep test logs out of the git working tree: many tests pass a relative
             # `log_dir` derived from `request.node.name`, which otherwise creates a large
             # number of untracked directories under the repo root during pytest runs.
-            if not os.path.isabs(self.log_dir):
-                log_root = os.environ.get(
-                    "DYN_TEST_OUTPUT_PATH",
-                    os.path.join(tempfile.gettempdir(), "dynamo_tests"),
-                )
-                self.log_dir = os.path.join(log_root, self.log_dir)
+            self.log_dir = resolve_test_output_path(self.log_dir)
 
             os.makedirs(self.log_dir, exist_ok=True)
             log_name = f"{self._command_name}.log.txt"

tests/utils/test_output.py

+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Utility for resolving test output paths.
+
+This module provides centralized logic for determining where test output
+(logs, temporary files, etc.) should be written. The goal is to keep test
+output out of the git working tree by default.
+"""
+
+import os
+import tempfile
+from pathlib import Path
+from typing import Union
+
+
+def resolve_test_output_path(path: Union[str, Path]) -> str:
+    """Resolve a test output path to an absolute path.
+
+    This function ensures test output is written to a dedicated location
+    rather than cluttering the git working tree. The behavior matches
+    the existing ManagedProcess implementation.
+
+    Args:
+        path: A relative or absolute path for test output.
+
+    Returns:
+        An absolute path. If the input is already absolute, it's returned
+        unchanged. If relative, it's resolved under the test output root
+        directory.
+
+    Environment Variables:
+        DYN_TEST_OUTPUT_PATH: Override the default test output root.
+                              Defaults to /tmp/dynamo_tests/.
+
+    Examples:
+        >>> resolve_test_output_path("/absolute/path")
+        '/absolute/path'
+        >>> resolve_test_output_path("test_foo")  # doctest: +SKIP
+        '/tmp/dynamo_tests/test_foo'
+    """
+    path_str = str(path)
+
+    if os.path.isabs(path_str):
+        return path_str
+
+    log_root = os.environ.get(
+        "DYN_TEST_OUTPUT_PATH",
+        os.path.join(tempfile.gettempdir(), "dynamo_tests"),
+    )
+    return os.path.join(log_root, path_str)