dynamo_check.py

#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""
dynamo package checker, Python import tester, and usage guide.

Combines version checking, import testing, and usage examples into a single tool.
Features dynamic component discovery and comprehensive troubleshooting guidance.

Usage:
    dynamo_check.py                        # Run all checks
    dynamo_check.py --import-check-only    # Only test imports
    dynamo_check.py --examples             # Only show examples
    dynamo_check.py --try-pythonpath      # Test imports with workspace paths
    dynamo_check.py --help                 # Show help

Outputs:
Dynamo Environment ($HOME/dynamo):
└─ Runtime components (ai-dynamo-runtime 0.4.0):
   ├─ /opt/dynamo/venv/lib/python3.12/site-packages/ai_dynamo_runtime-0.4.0.dist-info (created: 2025-08-12 15:10:05 PDT)
   ├─ /opt/dynamo/venv/lib/python3.12/site-packages/ai_dynamo_runtime.pth (modified: 2025-08-12 15:10:05 PDT)
      └─ Points to: $HOME/dynamo/lib/bindings/python/src
   ├─ ✅ dynamo._core        $HOME/dynamo/lib/bindings/python/src/dynamo/_core.cpython-312-x86_64-linux-gnu.so (modified: 2025-08-12 15:10:05 PDT)
   ├─ ✅ dynamo.nixl_connect $HOME/dynamo/lib/bindings/python/src/dynamo/nixl_connect/__init__.py
   ├─ ✅ dynamo.llm          $HOME/dynamo/lib/bindings/python/src/dynamo/llm/__init__.py
   └─ ✅ dynamo.runtime      $HOME/dynamo/lib/bindings/python/src/dynamo/runtime/__init__.py
└─ Framework components (ai-dynamo - Not installed):
   ├─ ❌ dynamo.frontend     No module named 'dynamo.frontend'
   ├─ ✅ dynamo.planner      $HOME/dynamo/components/planner/src/dynamo/planner/__init__.py
   ├─ ❌ dynamo.mocker       No module named 'dynamo.mocker'
   ├─ ❌ dynamo.trtllm       No module named 'dynamo.trtllm'
   ├─ ❌ dynamo.vllm         No module named 'dynamo.vllm'
   ├─ ❌ dynamo.sglang       No module named 'dynamo.sglang'
   └─ ❌ dynamo.llama_cpp    No module named 'dynamo.llama_cpp'
└─ Cargo home directory: $HOME/dynamo/.build/.cargo (CARGO_HOME is set)
└─ Cargo target directory: $HOME/dynamo/.build/target (CARGO_TARGET_DIR is set)
   ├─ Debug:   $HOME/dynamo/.build/target/debug (modified: 2025-08-12 15:10:02 PDT)
   └─ Binary:  $HOME/dynamo/.build/target/debug/libdynamo_llm_capi.so (modified: 2025-08-12 15:08:33 PDT)

Missing framework components. You can choose one of the following options:
1. For local development, set the PYTHONPATH environment variable:
   dynamo_check.py --try-pythonpath --import-check-only
   export PYTHONPATH="$HOME/dynamo/components/router/src:$HOME/dynamo/components/metrics/src:$HOME/dynamo/components/frontend/src:$HOME/dynamo/components/planner/src:$HOME/dynamo/components/backends/mocker/src:$HOME/dynamo/components/backends/trtllm/src:$HOME/dynamo/components/backends/vllm/src:$HOME/dynamo/components/backends/sglang/src:$HOME/dynamo/components/backends/llama_cpp/src"
2. For a production-release (slower build time), build the packages with:
   dynamo_build.sh --release
"""

import argparse
import datetime
import importlib.metadata
import json
import logging
import os
import platform
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from zoneinfo import ZoneInfo


class DynamoChecker:
    """Comprehensive dynamo package checker."""

    def __init__(self, workspace_dir: Optional[str] = None):
        # If a path is provided, use it directly; otherwise discover
        self.workspace_dir = (
            os.path.abspath(workspace_dir) if workspace_dir else self._find_workspace()
        )
        self.results: Dict[str, Any] = {}
        self._suppress_planner_warnings()
        self.clear_cuda_memory: bool = False
        # Collect warnings that should be printed later (after specific headers)
        self._deferred_messages: List[str] = []

    def _suppress_planner_warnings(self):
        """Suppress Prometheus endpoint warnings from planner module during import testing."""
        # The planner module logs a warning about Prometheus endpoint when imported
        # outside of a Kubernetes cluster. Suppress this for cleaner output.
        planner_logger = logging.getLogger("dynamo.planner.defaults")
        planner_logger.setLevel(logging.ERROR)

    # ====================================================================
    # WORKSPACE AND COMPONENT DISCOVERY
    # ====================================================================

    def _find_workspace(self) -> str:
        """Find dynamo workspace directory.

        Returns:
            Path to workspace directory or empty string if not found
            Example: '.' (if current dir), '/home/ubuntu/dynamo', '/workspace', or ''

        Note: Checks local path first, then common locations. Validates by looking for README.md file.
        """
        candidates = [
            ".",  # Current directory (local path)
            os.path.expanduser("~/dynamo"),
            "/workspace",
            "/home/ubuntu/dynamo",
        ]

        for candidate in candidates:
            if self._is_dynamo_workspace(candidate):
                # Always return absolute path for consistent $HOME replacement
                return os.path.abspath(candidate)
        return ""

    def _is_dynamo_workspace(self, path: str) -> bool:
        """Check if a directory is a dynamo workspace by looking for characteristic files/directories.

        Args:
            path: Directory path to check

        Returns:
            True if directory appears to be a dynamo workspace

        Note: Checks for multiple indicators like README.md, components/, lib/bindings/, lib/runtime/, Cargo.toml, etc.
        """
        if not os.path.exists(path):
            return False

        # Check for characteristic dynamo workspace files and directories
        indicators = [
            "README.md",
            "components",
            "lib/bindings/python",
            "lib/runtime",
            "Cargo.toml",
        ]

        # Require at least 3 indicators to be confident it's a dynamo workspace
        found_indicators = 0
        for indicator in indicators:
            if os.path.exists(os.path.join(path, indicator)):
                found_indicators += 1

        return found_indicators >= 4

    def _discover_runtime_components(self) -> List[str]:
        """Discover ai-dynamo-runtime components from filesystem.

        Returns:
            List of runtime component module names
            Example: ['dynamo._core', 'dynamo.nixl_connect', 'dynamo.llm', 'dynamo.runtime']

        Note: Always includes 'dynamo._core' (compiled Rust module), then scans
              lib/bindings/python/src/dynamo/ for additional components.
        """
        components = ["dynamo._core"]  # Always include compiled Rust module

        if not self.workspace_dir:
            return components

        # Scan runtime components (llm, runtime, nixl_connect, etc.)
        # Examples: lib/bindings/python/src/dynamo/{llm,runtime,nixl_connect}/__init__.py
        runtime_path = f"{self.workspace_dir}/lib/bindings/python/src/dynamo"
        if not os.path.exists(runtime_path):
            print(
                f"⚠️  Warning: Runtime components directory not found: {runtime_path}"
            )
            return components

        for item in os.listdir(runtime_path):
            item_path = os.path.join(runtime_path, item)
            if os.path.isdir(item_path) and os.path.exists(f"{item_path}/__init__.py"):
                components.append(f"dynamo.{item}")

        return components

    def _discover_framework_components(self) -> List[str]:
        """Discover ai-dynamo framework components from filesystem.

        Returns:
            List of framework component module names
            Example: ['dynamo.frontend', 'dynamo.planner', 'dynamo.vllm', 'dynamo.sglang', 'dynamo.llama_cpp']

        Note: Scans components/ and components/backends/ directories for modules with __init__.py files.
        """
        components: List[str] = []

        if not self.workspace_dir:
            return components

        # Scan direct components (frontend, planner, etc.)
        # Examples: components/{frontend,planner}/src/dynamo/{frontend,planner}/__init__.py
        comp_path = f"{self.workspace_dir}/components"
        if os.path.exists(comp_path):
            for item in os.listdir(comp_path):
                item_path = os.path.join(comp_path, item)
                if os.path.isdir(item_path) and os.path.exists(
                    f"{item_path}/src/dynamo/{item}/__init__.py"
                ):
                    components.append(f"dynamo.{item}")
        else:
            # Defer this message to print under the Dynamo header for alignment
            self._deferred_messages.append(
                f"⚠️  Warning: Components directory not found: {self._replace_home_with_var(comp_path)}"
            )

        # Scan backend components (vllm, sglang, etc.)
        # Examples: components/backends/{vllm,sglang,llama_cpp}/src/dynamo/{vllm,sglang,llama_cpp}/__init__.py
        backend_path = f"{self.workspace_dir}/components/backends"
        if os.path.exists(backend_path):
            for item in os.listdir(backend_path):
                item_path = os.path.join(backend_path, item)
                if os.path.isdir(item_path) and os.path.exists(
                    f"{item_path}/src/dynamo/{item}/__init__.py"
                ):
                    components.append(f"dynamo.{item}")
        else:
            # Defer this message to print under the Dynamo header for alignment
            self._deferred_messages.append(
                f"⚠️  Warning: Backend components directory not found: {self._replace_home_with_var(backend_path)}"
            )

        return components

    def _is_dynamo_build_available(self) -> bool:
        """Check if dynamo_build.sh is available in the same directory as this script.

        Returns:
            True if dynamo_build.sh exists in the same directory as dynamo_check.py
        """
        script_dir = Path(__file__).parent
        dynamo_build_path = script_dir / "dynamo_build.sh"
        return dynamo_build_path.exists()

    def _replace_home_with_var(self, path: str) -> str:
        """Replace user's home directory in path with $HOME.

        Args:
            path: File system path or colon-separated paths (for PYTHONPATH)

        Returns:
            Path with home directory replaced by $HOME if applicable
            Example: '/home/ubuntu/dynamo/...' -> '$HOME/dynamo/...'
            Example: '/home/ubuntu/dynamo/a:/home/ubuntu/dynamo/b' -> '$HOME/dynamo/a:$HOME/dynamo/b'
        """
        home_dir = os.path.expanduser("~")
        # Replace all occurrences for colon-separated paths like PYTHONPATH
        return path.replace(home_dir, "$HOME")

    def _format_timestamp_pdt(self, timestamp: float) -> str:
        """Format a timestamp in PDT timezone.

        Args:
            timestamp: Unix timestamp

        Returns:
            Formatted timestamp string in PDT or local timezone
            Example: '2025-08-10 22:22:52 PDT'
        """
        try:
            # Use zoneinfo (standard library in Python 3.9+)
            pdt = ZoneInfo("America/Los_Angeles")
            dt = datetime.datetime.fromtimestamp(timestamp, tz=pdt)
            return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
        except Exception:
            # Fallback to manual PDT offset approximation
            # PDT is UTC-7, so subtract 7 hours from UTC
            dt_utc = datetime.datetime.utcfromtimestamp(timestamp)
            dt_pdt = dt_utc - datetime.timedelta(hours=7)
            return dt_pdt.strftime("%Y-%m-%d %H:%M:%S PDT")

    def _get_cargo_info(self) -> Tuple[Optional[str], Optional[str]]:
        """Get cargo target directory and cargo home directory.

        Returns:
            Tuple of (target_directory, cargo_home) or (None, None) if cargo not available
            Example: ('/home/ubuntu/dynamo/.build/target', '/home/ubuntu/.cargo')
        """
        # First check if cargo is available
        try:
            subprocess.run(
                ["cargo", "--version"], capture_output=True, text=True, timeout=5
            )
        except (FileNotFoundError, subprocess.TimeoutExpired):
            # Do not print here; caller will render a nicely aligned warning
            return None, None

        # Get cargo home directory
        cargo_home = os.environ.get("CARGO_HOME")
        if not cargo_home:
            cargo_home = os.path.expanduser("~/.cargo")

        # Get cargo target directory
        target_directory = None
        try:
            # Run cargo metadata command to get target directory
            result = subprocess.run(
                ["cargo", "metadata", "--format-version=1", "--no-deps"],
                capture_output=True,
                text=True,
                timeout=10,
                cwd=self.workspace_dir
                if (self.workspace_dir and os.path.isdir(self.workspace_dir))
                else None,
            )

            if result.returncode == 0:
                # Parse JSON output to extract target_directory
                metadata = json.loads(result.stdout)
                target_directory = metadata.get("target_directory")
        except (
            subprocess.TimeoutExpired,
            subprocess.CalledProcessError,
            FileNotFoundError,
            json.JSONDecodeError,
        ):
            # cargo metadata failed or JSON parsing failed
            pass

        return target_directory, cargo_home

    def _print_system_info(self, clear_cuda: bool = False) -> bool:
        """Print concise system information as a top-level section.

        Tree structure:
        System info:
        ├─ Linux: ...
        ├─ GPU: ...
        └─ Python: ...
        """
        # OS info
        distro = ""
        version = ""
        try:
            os_release_path = "/etc/os-release"
            if os.path.exists(os_release_path):
                with open(os_release_path, "r") as f:
                    data = f.read()
                name = ""
                ver = ""
                for line in data.splitlines():
                    if line.startswith("NAME=") and not name:
                        name = line.split("=", 1)[1].strip().strip('"')
                    elif line.startswith("VERSION=") and not ver:
                        ver = line.split("=", 1)[1].strip().strip('"')
                distro = name
                version = ver
        except Exception:
            pass

        uname = platform.uname()
        # Memory (used/total) and CPU cores
        mem_used_gib = None
        mem_total_gib = None
        try:
            meminfo = {}
            with open("/proc/meminfo", "r") as f:
                for line in f:
                    if ":" in line:
                        k, v = line.split(":", 1)
                        meminfo[k.strip()] = v.strip()
            if "MemTotal" in meminfo and "MemAvailable" in meminfo:
                # Values are in kB
                total_kib = float(meminfo["MemTotal"].split()[0])
                avail_kib = float(meminfo["MemAvailable"].split()[0])
                used_kib = max(total_kib - avail_kib, 0.0)
                mem_total_gib = total_kib / (1024.0 * 1024.0)
                mem_used_gib = used_kib / (1024.0 * 1024.0)
        except Exception:
            pass

        cores = os.cpu_count() or 0

        if distro:
            base_linux = f"OS: {distro} {version} ({uname.system} {uname.release} {uname.machine})".strip()
        else:
            base_linux = (
                f"OS: {uname.system} {uname.release} {uname.version} ({uname.machine})"
            )

        extras = []
        if mem_used_gib is not None and mem_total_gib is not None:
            extras.append(f"Memory: {mem_used_gib:.1f}/{mem_total_gib:.1f} GiB")
        if cores:
            extras.append(f"Cores: {cores}")
        linux_line = base_linux if not extras else base_linux + "; " + "; ".join(extras)
        # Defer printing until we have all three lines; we print as a tree below

        # GPU info
        gpu_line = "GPU: none detected"
        gpu_driver_version: Optional[str] = None
        gpu_cuda_version: Optional[str] = None
        try:
            # Locate nvidia-smi robustly
            nvsmi = shutil.which("nvidia-smi")
            if not nvsmi:
                for candidate in [
                    "/usr/bin/nvidia-smi",
                    "/usr/local/bin/nvidia-smi",
                    "/usr/local/nvidia/bin/nvidia-smi",
                ]:
                    if os.path.exists(candidate) and os.access(candidate, os.X_OK):
                        nvsmi = candidate
                        break

            if nvsmi:
                # Fast list to count GPUs and get first name
                proc_list = subprocess.run(
                    [nvsmi, "-L"], capture_output=True, text=True, timeout=10
                )
                names: List[str] = []
                if proc_list.returncode == 0 and proc_list.stdout:
                    for line in proc_list.stdout.splitlines():
                        line = line.strip()
                        # Example: "GPU 0: NVIDIA A100-SXM4-40GB (UUID: GPU-...)"
                        if ":" in line:
                            part = line.split(":", 1)[1].strip()
                            # Take up to first parenthesis for clean model name
                            name_only = part.split("(")[0].strip()
                            names.append(name_only)

                # Query driver and CUDA
                driver = "?"
                cuda = "?"
                proc_q = subprocess.run(
                    [
                        nvsmi,
                        "--query-gpu=driver_version,cuda_version",
                        "--format=csv,noheader",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                if proc_q.returncode == 0 and proc_q.stdout.strip():
                    first = proc_q.stdout.strip().splitlines()[0].split(",")
                    if len(first) >= 1:
                        driver = first[0].strip()
                    if len(first) >= 2:
                        cuda = first[1].strip()
                else:
                    # Fallback: parse banner
                    proc_b = subprocess.run(
                        [nvsmi], capture_output=True, text=True, timeout=10
                    )
                    if proc_b.returncode == 0 and proc_b.stdout:
                        import re

                        m = re.search(r"Driver Version:\s*([0-9.]+)", proc_b.stdout)
                        if m:
                            driver = m.group(1)
                        m = re.search(r"CUDA Version:\s*([0-9.]+)", proc_b.stdout)
                        if m:
                            cuda = m.group(1)

                gpu_driver_version = driver
                gpu_cuda_version = cuda

                # Query power and memory usage/limits (first GPU)
                power_draw_w: Optional[str] = None
                power_limit_w: Optional[str] = None
                mem_used_mib: Optional[str] = None
                mem_total_mib: Optional[str] = None
                try:
                    proc_pm = subprocess.run(
                        [
                            nvsmi,
                            "--query-gpu=power.draw,power.limit,memory.used,memory.total",
                            "--format=csv,noheader,nounits",
                        ],
                        capture_output=True,
                        text=True,
                        timeout=10,
                    )
                    if proc_pm.returncode == 0 and proc_pm.stdout.strip():
                        first_pm = proc_pm.stdout.strip().splitlines()[0].split(",")
                        if len(first_pm) >= 1:
                            power_draw_w = first_pm[0].strip()
                        if len(first_pm) >= 2:
                            power_limit_w = first_pm[1].strip()
                        if len(first_pm) >= 3:
                            mem_used_mib = first_pm[2].strip()
                        if len(first_pm) >= 4:
                            mem_total_mib = first_pm[3].strip()
                except Exception:
                    pass

                power_mem_suffix = ""
                if any([power_draw_w, power_limit_w, mem_used_mib, mem_total_mib]):
                    # Build terse summary; include only available parts
                    parts = []
                    if power_draw_w or power_limit_w:
                        pd = power_draw_w if power_draw_w is not None else "?"
                        pl = power_limit_w if power_limit_w is not None else "?"
                        parts.append(f"Power: {pd}/{pl} W")
                    if mem_used_mib or mem_total_mib:
                        mu = mem_used_mib if mem_used_mib is not None else "?"
                        mt = mem_total_mib if mem_total_mib is not None else "?"
                        parts.append(f"Memory: {mu}/{mt} MiB")
                    power_mem_suffix = "; " + "; ".join(parts)

                if names:
                    gpu_count = len(names)
                    first_name = names[0]
                    if gpu_count == 1:
                        gpu_line = f"GPU: NVIDIA {first_name} (driver {driver}, CUDA {cuda}){power_mem_suffix}"
                    else:
                        gpu_line = f"GPU: NVIDIA x{gpu_count} ({first_name} first) (driver {driver}, CUDA {cuda}){power_mem_suffix}"
                else:
                    # No names but nvidia-smi present; still report driver/cuda
                    gpu_line = (
                        f"GPU: NVIDIA (driver {driver}, CUDA {cuda}){power_mem_suffix}"
                    )

            elif shutil.which("rocm-smi"):
                proc = subprocess.run(
                    ["rocm-smi", "-i"], capture_output=True, text=True, timeout=3
                )
                if proc.returncode == 0:
                    # Heuristic: count lines mentioning gfx or card
                    lines = proc.stdout.splitlines()
                    amd_gpus = [
                        line_text
                        for line_text in lines
                        if "Card" in line_text or "gfx" in line_text
                    ]
                    count = len(amd_gpus) if amd_gpus else 1
                    gpu_line = f"GPU: AMD ROCm x{count}"
            elif shutil.which("lspci"):
                proc = subprocess.run(
                    ["lspci"], capture_output=True, text=True, timeout=3
                )
                if proc.returncode == 0:
                    txt = proc.stdout.lower()
                    if "nvidia" in txt:
                        gpu_line = "GPU: NVIDIA (detected via lspci)"
                    elif "advanced micro devices" in txt or "amd" in txt:
                        gpu_line = "GPU: AMD (detected via lspci)"
                    elif "intel corporation" in txt and ("vga" in txt or "3d" in txt):
                        gpu_line = "GPU: Intel (detected via lspci)"
        except Exception:
            pass
        # Mark clearly when GPU not found
        if gpu_line == "GPU: none detected":
            gpu_line = "❌ " + gpu_line
        # Python info
        py_ver = platform.python_version()
        py_exec = sys.executable or "python"
        py_path_env = os.environ.get("PYTHONPATH")
        py_path_str = py_path_env if py_path_env else "unset"
        python_line = f"Python: {py_ver} ({py_exec}); PYTHONPATH={py_path_str}"
        if not os.path.exists(py_exec):
            python_line = "❌ Python: not found"

        # PyTorch info
        torch_version: Optional[str] = None
        try:
            import importlib

            torch = importlib.import_module("torch")  # type: ignore
            try:
                torch_version = getattr(torch, "__version__", None)  # type: ignore[attr-defined]
            except Exception:
                torch_version = None
        except Exception:
            # torch not installed
            pass

        # Optionally clear CUDA memory via torch
        extra_lines: List[str] = []
        if clear_cuda:
            status = "CUDA memory: torch not available"
            try:
                import importlib

                torch = importlib.import_module("torch")  # type: ignore
                if hasattr(torch, "cuda") and torch.cuda.is_available():
                    try:
                        torch.cuda.empty_cache()
                        if hasattr(torch.cuda, "reset_peak_memory_stats"):
                            torch.cuda.reset_peak_memory_stats()
                        status = "CUDA memory: cache cleared; peak stats reset"
                    except Exception as e:
                        status = (
                            f"CUDA memory: failed to clear ({e.__class__.__name__})"
                        )
                else:
                    status = "CUDA memory: CUDA not available"
            except Exception:
                pass
            extra_lines.append(status)

        # Prepare CUDA line (single, compact) and print System info in required order
        # Use driver/CUDA version from nvidia-smi when available
        cuda_line: Optional[str] = None
        if gpu_driver_version is not None or gpu_cuda_version is not None:
            d = gpu_driver_version if gpu_driver_version is not None else "unknown"
            c = gpu_cuda_version if gpu_cuda_version is not None else "unknown"
            cuda_line = f"CUDA: driver {d}, CUDA {c}"
        else:
            cuda_line = "❌ CUDA: not found"

        # Detect cargo binary path and version for heading
        cargo_path = shutil.which("cargo")
        cargo_version = None
        try:
            proc = subprocess.run(
                ["cargo", "--version"], capture_output=True, text=True, timeout=5
            )
            if proc.returncode == 0 and proc.stdout:
                cargo_version = proc.stdout.strip()
        except Exception:
            pass

        cargo_target, cargo_home = self._get_cargo_info()
        has_cargo = bool(cargo_path or cargo_home or cargo_target)

        print("System info:")
        # Linux
        print(f"├─ {linux_line}")
        # GPU
        print(f"├─ {gpu_line}")
        # CUDA right after GPU, if available (power/memory already appended to GPU line)
        if cuda_line:
            print(f"├─ {cuda_line}")
        # Python line; if more top-level entries come after Python subtree, use mid symbol
        more_after_python = bool(extra_lines or has_cargo)
        print(f"{'├─' if more_after_python else '└─'} {python_line}")
        # Torch version as a child under Python
        if torch_version:
            print("   └─ Torch: " + str(torch_version))
        else:
            # Show as a child under Python
            print("   └─ ❌ Torch: not installed")
        # Extra lines (e.g., CUDA memory clear status)
        for i, line in enumerate(extra_lines):
            # If cargo follows after extra lines, use mid symbol; else close on last
            is_last_extra = i == len(extra_lines) - 1
            symbol = "├─" if (has_cargo or not is_last_extra) else "└─"
            print(f"{symbol} {line}")

        # Cargo Info block
        if has_cargo:
            cargo_heading = "Cargo ("
            if cargo_path:
                cargo_heading += f"{cargo_path}"
            else:
                cargo_heading += "cargo not found"
            if cargo_version:
                cargo_heading += f", {cargo_version}"
            cargo_heading += ")"

            # Cargo heading is not the last top-level child (Dynamo Environment follows)
            print(f"├─ {cargo_heading}")

            # Under cargo heading, indent nested details
            if cargo_home:
                cargo_home_env = os.environ.get("CARGO_HOME")
                display_cargo_home = self._replace_home_with_var(cargo_home)
                if cargo_home_env:
                    print(
                        f"   ├─ Cargo home directory: {display_cargo_home} (CARGO_HOME is set)"
                    )
                else:
                    # If there's also a target below, keep mid connector, else close
                    print(
                        f"   {'├─' if cargo_target else '└─'} Cargo home directory: {display_cargo_home}"
                    )

            if cargo_target:
                cargo_target_env = os.environ.get("CARGO_TARGET_DIR")
                display_cargo_target = self._replace_home_with_var(cargo_target)
                target_msg = (
                    f"   └─ Cargo target directory: {display_cargo_target} (CARGO_TARGET_DIR is set)"
                    if cargo_target_env
                    else f"   └─ Cargo target directory: {display_cargo_target}"
                )
                print(target_msg)

                # Nested details under Cargo target directory
                debug_dir = os.path.join(cargo_target, "debug")
                release_dir = os.path.join(cargo_target, "release")

                debug_exists = os.path.exists(debug_dir)
                release_exists = os.path.exists(release_dir)

                # Find *.so file
                so_file = self._find_so_file(cargo_target)
                has_so_file = so_file is not None

                if debug_exists:
                    symbol = "├─" if release_exists or has_so_file else "└─"
                    display_debug_dir = self._replace_home_with_var(debug_dir)
                    try:
                        debug_mtime = os.path.getmtime(debug_dir)
                        debug_time = self._format_timestamp_pdt(debug_mtime)
                        print(
                            f"      {symbol} Debug:   {display_debug_dir} (modified: {debug_time})"
                        )
                    except OSError:
                        print(
                            f"      {symbol} Debug:   {display_debug_dir} (unable to read timestamp)"
                        )

                if release_exists:
                    symbol = "├─" if has_so_file else "└─"
                    display_release_dir = self._replace_home_with_var(release_dir)
                    try:
                        release_mtime = os.path.getmtime(release_dir)
                        release_time = self._format_timestamp_pdt(release_mtime)
                        print(
                            f"      {symbol} Release: {display_release_dir} (modified: {release_time})"
                        )
                    except OSError:
                        print(
                            f"      {symbol} Release: {display_release_dir} (unable to read timestamp)"
                        )

                if has_so_file and so_file is not None:
                    display_so_file = self._replace_home_with_var(so_file)
                    try:
                        so_mtime = os.path.getmtime(so_file)
                        so_time = self._format_timestamp_pdt(so_mtime)
                        print(
                            f"      └─ Binary:  {display_so_file} (modified: {so_time})"
                        )
                    except OSError:
                        print(
                            f"      └─ Binary:  {display_so_file} (unable to read timestamp)"
                        )
        else:
            # Cargo not found: show as a top-level sibling; Dynamo follows, so use mid connector
            print(
                "├─ ❌ Cargo: not found (install Rust toolchain to see cargo target directory)"
            )
        # Determine if any errors were printed in system info (treat only Python and Cargo as fatal here)
        system_errors_found = False
        if isinstance(python_line, str) and python_line.startswith("❌"):
            system_errors_found = True
        if not has_cargo:
            system_errors_found = True
        return system_errors_found

    def _find_so_file(self, target_directory: str) -> Optional[str]:
        """Find the compiled *.so file in target directory or Python bindings.

        Args:
            target_directory: Path to cargo target directory

        Returns:
            Path to *.so file or None if not found
            Example: '/home/ubuntu/dynamo/target/debug/libdynamo_core.so'
        """
        if not target_directory or not os.path.exists(target_directory):
            return None

        # Look for *.so files in debug and release directories
        for profile in ["debug", "release"]:
            profile_dir = os.path.join(target_directory, profile)
            if os.path.exists(profile_dir):
                try:
                    for root, dirs, files in os.walk(profile_dir):
                        for file in files:
                            if file.endswith(".so"):
                                return os.path.join(root, file)
                except OSError:
                    continue

        # Also check Python bindings directory for installed *.so
        if self.workspace_dir:
            bindings_dir = f"{self.workspace_dir}/lib/bindings/python/src/dynamo"
            if os.path.exists(bindings_dir):
                try:
                    for root, dirs, files in os.walk(bindings_dir):
                        for file in files:
                            if file.endswith(".so") and "_core" in file:
                                return os.path.join(root, file)
                except OSError:
                    pass

        return None

    def _get_cargo_build_profile(self, target_directory: str) -> Optional[str]:
        """Determine which cargo build profile (debug/release) was used most recently.

        Args:
            target_directory: Path to cargo target directory

        Returns:
            'debug', 'release', 'debug/release', or None if cannot determine
            Example: 'debug'
        """
        # First check environment variables that indicate current build profile
        profile_env = os.environ.get("PROFILE")
        if profile_env:
            if profile_env == "dev":
                return "debug"
            elif profile_env == "release":
                return "release"

        # Check OPT_LEVEL as secondary indicator
        opt_level = os.environ.get("OPT_LEVEL")
        if opt_level:
            if opt_level == "0":
                return "debug"
            elif opt_level in ["2", "3"]:
                return "release"

        # Fall back to filesystem inspection
        if not target_directory or not os.path.exists(target_directory):
            return None

        debug_dir = os.path.join(target_directory, "debug")
        release_dir = os.path.join(target_directory, "release")

        debug_exists = os.path.exists(debug_dir)
        release_exists = os.path.exists(release_dir)

        if not debug_exists and not release_exists:
            return None
        elif debug_exists and not release_exists:
            return "debug"
        elif release_exists and not debug_exists:
            return "release"
        else:
            # Both exist, check which was modified more recently
            try:
                debug_mtime = os.path.getmtime(debug_dir)
                release_mtime = os.path.getmtime(release_dir)

                if (
                    abs(debug_mtime - release_mtime) < 1.0
                ):  # Same timestamp (within 1 second)
                    return "debug/release"  # Both available, runtime choice depends on invocation
                else:
                    return "release" if release_mtime > debug_mtime else "debug"
            except OSError:
                return None

    def _setup_pythonpath(self):
        """Set up PYTHONPATH for component imports."""
        if not self.workspace_dir:
            return

        paths = []

        # Collect component source paths
        comp_path = f"{self.workspace_dir}/components"
        if os.path.exists(comp_path):
            for item in os.listdir(comp_path):
                src_path = f"{comp_path}/{item}/src"
                if os.path.exists(src_path):
                    paths.append(src_path)
        else:
            print(
                f"⚠️  Warning: Components directory not found for PYTHONPATH setup: {comp_path}"
            )

        # Collect backend source paths
        backend_path = f"{self.workspace_dir}/components/backends"
        if os.path.exists(backend_path):
            for item in os.listdir(backend_path):
                src_path = f"{backend_path}/{item}/src"
                if os.path.exists(src_path):
                    paths.append(src_path)
        else:
            print(
                f"⚠️  Warning: Backend components directory not found for PYTHONPATH setup: {backend_path}"
            )

        # Update sys.path for current process
        if paths:
            # Add paths to sys.path for immediate effect on imports
            for path in paths:
                if path not in sys.path:
                    sys.path.insert(0, path)  # Insert at beginning for priority

            # Show what PYTHONPATH would be (for manual shell setup)
            pythonpath_value = ":".join(paths)
            current_path = os.environ.get("PYTHONPATH", "")
            if current_path:
                pythonpath_value = f"{pythonpath_value}:{current_path}"

            print(
                f'Below are the results if you export PYTHONPATH="{pythonpath_value}":'
            )
            print(f"   ({len(paths)} workspace component paths found)")
            for path in paths:
                print(f"   • {path}")
            print()
        else:
            print("⚠️  Warning: No component source paths found for PYTHONPATH setup")

    # ====================================================================
    # IMPORT TESTING
    # ====================================================================

    def _test_component_group(
        self,
        components: List[str],
        package_name: str,
        group_name: str,
        max_width: int,
        site_packages: str,
        collect_failures: bool = False,
        package_info: Optional[Dict[str, Any]] = None,
        sub_indent: str = "   ",
    ) -> Tuple[Dict[str, str], List[str]]:
        """Test a group of components for a given package.

        Args:
            components: List of component names to test
                Example: ['dynamo._core', 'dynamo.llm', 'dynamo.runtime']
            package_name: Name of the package to get version from
                Example: 'ai-dynamo-runtime'
            group_name: Display name for the group
                Example: 'Runtime components'
            max_width: Maximum width for component name alignment
                Example: 20
            site_packages: Path to site-packages directory
                Example: '/opt/dynamo/venv/lib/python3.12/site-packages'
            collect_failures: Whether to collect failed component names
                Example: True (for framework components), False (for runtime)

        Returns:
            Tuple of (results dict, list of failed components)
            Example: ({'dynamo._core': '✅ Success', 'dynamo.llm': '❌ Failed: No module named dynamo.llm'},
                     ['dynamo.llm'])

        Output printed to console:
            Dynamo Environment ($HOME/dynamo):
            └─ Runtime components (ai-dynamo-runtime 0.4.0):
               ├─ /opt/dynamo/venv/lib/.../ai_dynamo_runtime-0.4.0.dist-info (created: 2025-08-12 14:17:34 PDT)
               ├─ ✅ dynamo._core        /opt/dynamo/venv/lib/.../dynamo/_core.cpython-312-x86_64-linux-gnu.so
               └─ ❌ dynamo.llm          No module named 'dynamo.llm'
        """
        results = {}
        failures = []

        # Print header with version info
        try:
            version = importlib.metadata.version(package_name)
            header = f"{group_name} ({package_name} {version}):"
        except importlib.metadata.PackageNotFoundError:
            header = f"{group_name} ({package_name} - Not installed):"
        except Exception:
            header = f"{group_name} ({package_name}):"

        print(header)

        # Determine if package info should use ├─ or └─ based on whether there are components
        has_components = len(components) > 0
        package_symbol = "├─" if has_components else "└─"

        # Print package info as subitem of component group (only if found)
        if package_info:
            package_path = package_info.get("path", "")
            package_created = package_info.get("created", "")
            display_path = self._replace_home_with_var(package_path)
            if package_created:
                print(
                    f"{sub_indent}{package_symbol} {display_path} (created: {package_created})"
                )
            else:
                print(f"{sub_indent}{package_symbol} {display_path}")

            # Show .pth files if they exist (editable installs) - at same level as package info
            pth_files = package_info.get("pth_files", [])
            for i, pth_file in enumerate(pth_files):
                is_last_pth = i == len(pth_files) - 1
                pth_symbol = "└─" if (is_last_pth and not has_components) else "├─"
                display_pth_path = self._replace_home_with_var(pth_file["path"])
                display_points_to = self._replace_home_with_var(pth_file["points_to"])
                print(
                    f"{sub_indent}{pth_symbol} {display_pth_path} (modified: {pth_file['modified']})"
                )
                print(f"{sub_indent}   └─ Points to: {display_points_to}")
        # Don't print anything for "Not found" - just skip it

        # Test each component as subitems of the package
        for i, component in enumerate(components):
            # Determine tree symbol - last component gets └─, others get ├─, with proper indentation (deeper nesting)
            is_last = i == len(components) - 1
            tree_symbol = f"{sub_indent}{'└─' if is_last else '├─'}"

            try:
                module = __import__(component, fromlist=[""])
                results[component] = "✅ Success"
                # Get module path for location info
                module_path = getattr(module, "__file__", "built-in")
                if module_path and module_path != "built-in":
                    # Only show timestamps for generated files (*.so, *.pth, etc.), not __init__.py
                    timestamp_str = ""
                    show_timestamp = False

                    # Check if this is a generated file we want to show timestamps for
                    if any(
                        module_path.endswith(ext)
                        for ext in [".so", ".pth", ".dll", ".dylib"]
                    ):
                        show_timestamp = True

                    if show_timestamp:
                        try:
                            if os.path.exists(module_path):
                                mtime = os.path.getmtime(module_path)
                                timestamp_str = (
                                    f" (modified: {self._format_timestamp_pdt(mtime)})"
                                )
                        except OSError:
                            pass

                    if self.workspace_dir and module_path.startswith(
                        self.workspace_dir
                    ):
                        # From workspace source
                        rel_path = os.path.relpath(module_path, self.workspace_dir)
                        if show_timestamp:
                            print(
                                f"{tree_symbol} ✅ {component:<{max_width}} {rel_path}{timestamp_str}"
                            )
                        else:
                            print(
                                f"{tree_symbol} ✅ {component:<{max_width}} {rel_path}"
                            )
                    elif site_packages and module_path.startswith(site_packages):
                        # From installed package - show path with $HOME replacement
                        display_path = self._replace_home_with_var(module_path)
                        if show_timestamp:
                            print(
                                f"{tree_symbol} ✅ {component:<{max_width}} {display_path}{timestamp_str}"
                            )
                        else:
                            print(
                                f"{tree_symbol} ✅ {component:<{max_width}} {display_path}"
                            )
                    else:
                        # Other location - show path with $HOME replacement
                        display_path = self._replace_home_with_var(module_path)
                        if show_timestamp:
                            print(
                                f"{tree_symbol} ✅ {component:<{max_width}} {display_path}{timestamp_str}"
                            )
                        else:
                            print(
                                f"{tree_symbol} ✅ {component:<{max_width}} {display_path}"
                            )
                else:
                    built_in_suffix = (
                        " (built-in)"
                        if group_name.lower().startswith("framework")
                        else " built-in"
                    )
                    print(f"{tree_symbol} ✅ {component:<{max_width}}{built_in_suffix}")
            except ImportError as e:
                results[component] = f"❌ Failed: {e}"
                print(f"{tree_symbol} ❌ {component:<{max_width}} {e}")
                if collect_failures:
                    failures.append(component)

        return results, failures

    def _get_package_info(self, package_name: str) -> Dict[str, Any]:
        """Get package installation information including .pth files.

        Args:
            package_name: Name of the package (e.g., 'ai-dynamo-runtime')

        Returns:
            Dict with 'path', 'created', and optionally 'pth_files' keys
        """
        import site

        site_packages_dirs = site.getsitepackages()
        if hasattr(site, "getusersitepackages"):
            site_packages_dirs.append(site.getusersitepackages())

        result: Dict[str, Any] = {}
        pth_files: List[Dict[str, str]] = []

        for site_dir in site_packages_dirs:
            if not os.path.exists(site_dir):
                continue

            try:
                for file in os.listdir(site_dir):
                    # Look for .dist-info directories that exactly match the package name
                    if file.endswith(".dist-info"):
                        # Extract package name from .dist-info directory name
                        dist_name = file.replace(".dist-info", "")
                        # Handle version suffixes (e.g., ai_dynamo_runtime-0.4.0 -> ai_dynamo_runtime)
                        base_name = (
                            dist_name.split("-")[0] if "-" in dist_name else dist_name
                        )
                        expected_name = package_name.replace("-", "_")

                        if base_name == expected_name:
                            dist_info_path = os.path.join(site_dir, file)
                            if os.path.isdir(dist_info_path):
                                try:
                                    ctime = os.path.getctime(dist_info_path)
                                    created_time = self._format_timestamp_pdt(ctime)
                                    result.update(
                                        {
                                            "path": dist_info_path,
                                            "created": created_time,
                                        }
                                    )
                                except OSError:
                                    result.update({"path": dist_info_path})

                    # Look for .pth files that match this specific package
                    if file.endswith(".pth"):
                        # Match .pth files to specific packages
                        pth_matches_package = False
                        if package_name == "ai-dynamo-runtime":
                            # Look for ai_dynamo_runtime.pth or similar
                            if (
                                "ai_dynamo_runtime" in file.lower()
                                or file.lower().startswith("ai_dynamo_runtime")
                            ):
                                pth_matches_package = True
                        elif package_name == "ai-dynamo":
                            # Look for _ai_dynamo.pth or ai_dynamo.pth (but not ai_dynamo_runtime.pth)
                            if (
                                "ai_dynamo" in file.lower()
                                or "_ai_dynamo" in file.lower()
                            ) and "runtime" not in file.lower():
                                pth_matches_package = True

                        if pth_matches_package:
                            pth_path = os.path.join(site_dir, file)
                            try:
                                mtime = os.path.getmtime(pth_path)
                                # Read the content to see what path it adds
                                with open(pth_path, "r") as f:
                                    content = f.read().strip()
                                pth_files.append(
                                    {
                                        "path": pth_path,
                                        "modified": self._format_timestamp_pdt(mtime),
                                        "points_to": content,
                                    }
                                )
                            except OSError:
                                pass
            except OSError:
                continue

        if pth_files:
            result["pth_files"] = pth_files

        return result

    def test_imports(self) -> Dict[str, str]:
        """Test imports for all discovered components.

        Returns:
            Dictionary mapping component names to their import status
            Example: {
                'dynamo._core': '✅ Success',
                'dynamo.llm': '✅ Success',
                'dynamo.runtime': '✅ Success',
                'dynamo.frontend': '❌ Failed: No module named dynamo.frontend',
                'dynamo.planner': '✅ Success'
            }

        Console output example:
            Dynamo Environment ($HOME/dynamo):
            └─ Runtime components (ai-dynamo-runtime 0.4.0):
               ├─ /opt/dynamo/venv/lib/.../ai_dynamo_runtime-0.4.0.dist-info (created: 2025-08-12 14:17:34 PDT)
               ├─ /opt/dynamo/venv/lib/.../ai_dynamo_runtime.pth (modified: 2025-08-12 14:17:34 PDT)
                  └─ Points to: $HOME/dynamo/lib/bindings/python/src
               ├─ ✅ dynamo._core        /opt/dynamo/venv/lib/.../dynamo/_core.cpython-312-x86_64-linux-gnu.so
               └─ ✅ dynamo.llm          /opt/dynamo/venv/lib/.../dynamo/llm/__init__.py

            └─ Framework components (ai-dynamo - Not installed):
               ├─ ✅ dynamo.frontend     /opt/dynamo/venv/lib/.../dynamo/frontend/__init__.py
               └─ ❌ dynamo.missing      No module named 'dynamo.missing'
        """
        results = {}

        # Print system info at top-level, before Dynamo Environment
        system_errors = self._print_system_info(clear_cuda=self.clear_cuda_memory)

        # Then print main environment header as a subtree under System info
        if (
            self.workspace_dir
            and os.path.exists(self.workspace_dir)
            and self._is_dynamo_workspace(self.workspace_dir)
        ):
            workspace_path = os.path.abspath(self.workspace_dir)
            display_workspace = self._replace_home_with_var(workspace_path)
            print(f"└─ Dynamo ({display_workspace}):")
            # Backend components directory warning after the Dynamo line
            backend_path = f"{self.workspace_dir}/components/backends"
            if not os.path.exists(backend_path):
                print(
                    f"   ⚠️  Warning: Backend components directory not found: {self._replace_home_with_var(backend_path)}"
                )
            # If there were deferred messages (e.g., invalid --path), show them here under Dynamo
            for message in self._deferred_messages:
                print(f"   {message}")
        else:
            # If a user provided an invalid --path, reflect that, otherwise generic not found
            if self.workspace_dir and not os.path.exists(self.workspace_dir):
                print(f"└─ Dynamo ({self._replace_home_with_var(self.workspace_dir)}):")
                print("   ❌ Workspace path does not exist")
            elif self.workspace_dir and not self._is_dynamo_workspace(
                self.workspace_dir
            ):
                print(f"└─ Dynamo ({self._replace_home_with_var(self.workspace_dir)}):")
                print("   ❌ Invalid dynamo workspace (missing expected files)")
            else:
                print("└─ Dynamo (workspace not found):")

        # Discover all components
        runtime_components = self._discover_runtime_components()
        framework_components = self._discover_framework_components()

        # Calculate max width for alignment across ALL components
        all_components = runtime_components + framework_components
        max_width = max(len(comp) for comp in all_components) if all_components else 0

        # Get site-packages path for comparison
        import site

        site_packages = site.getsitepackages()[0] if site.getsitepackages() else ""

        # Get package information for headers
        runtime_package_info = self._get_package_info("ai-dynamo-runtime")
        framework_package_info = self._get_package_info("ai-dynamo")

        # Test runtime components (as subitem of Dynamo Environment, indented; components printed below group header)
        runtime_results, _ = self._test_component_group(
            runtime_components,
            "ai-dynamo-runtime",
            "   └─ Runtime components",
            max_width,
            site_packages,
            collect_failures=False,
            package_info=runtime_package_info,
            sub_indent="      ",
        )
        results.update(runtime_results)

        # Test framework components (as subitem of Dynamo Environment, indented; components printed below group header)
        framework_results, framework_failures = self._test_component_group(
            framework_components,
            "ai-dynamo",
            "   └─ Framework components",
            max_width,
            site_packages,
            collect_failures=True,
            package_info=framework_package_info,
            sub_indent="      ",
        )
        results.update(framework_results)

        # Cargo information is printed under System info

        # Show PYTHONPATH recommendation if any framework components failed (moved to end)
        if framework_failures and self.workspace_dir:
            pythonpath = self._get_pythonpath()
            if pythonpath:
                # Apply $HOME replacement to PYTHONPATH for consistency
                display_pythonpath = self._replace_home_with_var(pythonpath)
                print(
                    "\nMissing framework components. You can choose one of the following options:"
                )
                print(
                    "1. For local development, set the PYTHONPATH environment variable:"
                )
                print(
                    f'   dynamo_check.py --try-pythonpath --import-check-only\n   export PYTHONPATH="{display_pythonpath}"'
                )
                not_found_suffix = (
                    ""
                    if self._is_dynamo_build_available()
                    else "  # (dynamo_build.sh not found)"
                )
                print(
                    "2. For a production-release (slower build time), build the packages with:"
                )
                print(f"   dynamo_build.sh --release{not_found_suffix}")

        # Exit with non-zero status if any errors detected
        # Treat Python or Cargo failures from system info, and invalid path, as failures.
        any_failures = (
            system_errors
            or any(msg.startswith("❌ Error:") for msg in self._deferred_messages)
            or bool(framework_failures)
        )
        # Store whether errors occurred for overall run
        self.results["had_errors"] = any_failures
        return results

    # ====================================================================
    # USAGE EXAMPLES AND GUIDANCE
    # ====================================================================

    def show_usage_examples(self):
        """Show practical usage examples.

        Prints formatted examples of common dynamo operations including:
        - Starting frontend server
        - Starting vLLM backend
        - Making inference requests
        - Setting up development environment
        - Building packages

        Console output example:
            Usage Examples
            ========================================

            1. Start Frontend Server:
               python -m dynamo.frontend --http-port 8000

            2. Start vLLM Backend:
               python -m dynamo.vllm --model Qwen/Qwen2.5-0.5B
               ...
        """
        print(
            """
Usage Examples
========================================

1. Start Frontend Server:
   python -m dynamo.frontend --http-port 8000

2. Start vLLM Backend:
   python -m dynamo.vllm --model Qwen/Qwen2.5-0.5B

3. Send Inference Request:
   curl -X POST http://localhost:8000/v1/completions \\
        -H 'Content-Type: application/json' \\
        -d '{"model": "Qwen/Qwen2.5-0.5B", "prompt": "Hello", "max_tokens": 50}'

4. For local development: Set PYTHONPATH to use workspace sources without rebuilding:
   • Discover what PYTHONPATH to set: dynamo_check.py --try-pythonpath --import-check-only"""
        )
        if self.workspace_dir:
            pythonpath = self._get_pythonpath()
            display_pythonpath = self._replace_home_with_var(pythonpath)
            print(
                f'   • Then set in your shell: export PYTHONPATH="{display_pythonpath}"'
            )
        else:
            print(
                '   • Then set in your shell: export PYTHONPATH="$HOME/dynamo/components/*/src"'
            )

        not_found_suffix = (
            "" if self._is_dynamo_build_available() else " (dynamo_build.sh not found)"
        )
        print(
            f"""
5. Build Packages:
   dynamo_build.sh --dev              # Development mode{not_found_suffix}
   dynamo_build.sh --release          # Production wheels{not_found_suffix}"""
        )

    def _get_pythonpath(self) -> str:
        """Generate PYTHONPATH recommendation string.

        Returns:
            Colon-separated string of component source paths
            Example: '/home/ubuntu/dynamo/components/frontend/src:/home/ubuntu/dynamo/components/planner/src:/home/ubuntu/dynamo/components/backends/vllm/src'

        Note: Scans workspace for all component src directories and joins them for PYTHONPATH usage.
        """
        paths = []
        if not self.workspace_dir:
            return ""

        # Collect all component source paths
        comp_path = f"{self.workspace_dir}/components"
        if os.path.exists(comp_path):
            for item in os.listdir(comp_path):
                src_path = f"{comp_path}/{item}/src"
                if os.path.exists(src_path):
                    paths.append(src_path)

        # Collect all backend source paths
        backend_path = f"{self.workspace_dir}/components/backends"
        if os.path.exists(backend_path):
            for item in os.listdir(backend_path):
                src_path = f"{backend_path}/{item}/src"
                if os.path.exists(src_path):
                    paths.append(src_path)

        return ":".join(paths)

    # ====================================================================
    # TROUBLESHOOTING AND SUMMARY
    # ====================================================================

    def show_troubleshooting(self):
        """Troubleshooting section removed for terse output."""
        return

    def show_summary(self):
        """Summary output intentionally omitted for terse mode."""
        return

    # ====================================================================
    # MAIN ORCHESTRATION
    # ====================================================================

    def run_all(self):
        """Run comprehensive check with all functionality.

        Performs complete dynamo package validation including:
        - Component discovery and import testing
        - Usage examples and troubleshooting guidance
        - Summary of results

        Console output: terse, tree-formatted sections
        """
        # Terse mode: no banner or separators

        # Execute all checks (package versions now shown in import testing headers)
        self.results["imports"] = self.test_imports()

        # Check if there were any import failures
        import_results = self.results.get("imports", {})
        has_failures = any(result.startswith("❌") for result in import_results.values())

        # Provide guidance (show examples only if all checks succeed and no errors flagged)
        had_errors_flag = bool(self.results.get("had_errors"))
        if not has_failures and not had_errors_flag:
            self.show_usage_examples()
        self.show_troubleshooting()
        self.show_summary()
        # If any errors found, exit with status 1
        had_errors = bool(self.results.get("had_errors"))
        if had_errors:
            sys.exit(1)


def main():
    """Main function with command line argument parsing."""
    parser = argparse.ArgumentParser(description="Comprehensive dynamo package checker")
    parser.add_argument(
        "--import-check-only", action="store_true", help="Only test imports"
    )
    parser.add_argument("--examples", action="store_true", help="Only show examples")
    parser.add_argument(
        "--try-pythonpath",
        action="store_true",
        help="Test imports with workspace component source directories in sys.path",
    )
    parser.add_argument(
        "--path",
        type=str,
        default=None,
        help="Explicit path to dynamo workspace; if set, bypass workspace auto-discovery",
    )
    parser.add_argument(
        "--clear-cuda-memory",
        action="store_true",
        help="Attempt to clear CUDA cache and reset peak memory stats via torch",
    )

    args = parser.parse_args()
    checker = DynamoChecker(workspace_dir=args.path)
    # If --path is provided, validate it; do not exit early, but record error to display and for exit code
    if args.path:
        abs_path = os.path.abspath(args.path)
        if (not os.path.exists(abs_path)) or (
            not checker._is_dynamo_workspace(abs_path)
        ):
            checker._deferred_messages.append(
                f"❌ Error: invalid workspace path: {abs_path}"
            )
    checker.clear_cuda_memory = bool(args.clear_cuda_memory)

    # Set up sys.path if requested
    if args.try_pythonpath:
        checker._setup_pythonpath()

    if args.import_check_only:
        checker.test_imports()
        # Exit code handled inside run; reflect errors if set
        had_errors = bool(checker.results.get("had_errors"))
        if had_errors:
            sys.exit(1)
    elif args.examples:
        # Always show system info first, then environment header
        checker._print_system_info(clear_cuda=checker.clear_cuda_memory)
        if checker.workspace_dir:
            workspace_path = os.path.abspath(checker.workspace_dir)
            display_workspace = checker._replace_home_with_var(workspace_path)
            print(f"Dynamo ({display_workspace}):")
        else:
            print("Dynamo (workspace not found):")
        checker.show_usage_examples()
    else:
        checker.run_all()


if __name__ == "__main__":
    main()