refactor: Dev Container (devcontainer) to standardize paths to /workspace (for...

refactor: Dev Container (devcontainer) to standardize paths to /workspace (for pytest compatibility) (#3870)
Signed-off-by: Keiven Chang <keivenchang@users.noreply.github.com>

.devcontainer/devcontainer.json.j2

@@ -29,7 +29,7 @@
                 // Disable automatic copying of .gitconfig to avoid errors
                 "dev.containers.copyGitConfig": false,
                 "terminal.integrated.defaultProfile.linux": "bash",
-                "terminal.integrated.cwd": "/home/ubuntu/dynamo",
+                "terminal.integrated.cwd": "/workspace",
 
                 "python.defaultInterpreterPath": "/opt/dynamo/venv/bin/python",
                 "python.linting.enabled": true,
@@ -38,7 +38,7 @@
                 "rust-analyzer.checkOnSave.command": "clippy",
                 "rust-analyzer.checkOnSave.enable": true,
                 "rust-analyzer.cargo.buildScripts.enable": true,
-                "rust-analyzer.cargo.targetDir": "/home/ubuntu/dynamo/.build/target",
+                "rust-analyzer.cargo.targetDir": "/workspace/target",
                 "rust-analyzer.procMacro.enable": true,
                 "rust-analyzer.completion.autoimport.enable": true,
 
@@ -56,16 +56,10 @@
             }
         }
     },
-    "workspaceFolder": "/home/ubuntu/dynamo",
-    "workspaceMount": "source=${localWorkspaceFolder},target=/home/ubuntu/dynamo,type=bind,consistency=cached",
+    "workspaceFolder": "/workspace",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
     "userEnvProbe": "interactiveShell",
-    "postCreateCommand": "/bin/bash /home/ubuntu/dynamo/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
-    "containerEnv": {
-        "DYNAMO_HOME": "/home/ubuntu/dynamo",
-        "CARGO_HOME": "/home/ubuntu/dynamo/.build/.cargo",
-        "RUSTUP_HOME": "/home/ubuntu/dynamo/.build/.rustup",
-        "CARGO_TARGET_DIR": "/home/ubuntu/dynamo/.build/target"
-    },
+    "postCreateCommand": "/bin/bash /workspace/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
     "remoteEnv": {
         // Optional convenience tokens passed from host. SSH agent is forwarded via your IDE setting, not here by default.
         "GITHUB_TOKEN": "${localEnv:GITHUB_TOKEN}",

.devcontainer/post-create.sh

@@ -4,8 +4,9 @@
 
 set -eu
 
-# Set DYNAMO_HOME if not already set
-export DYNAMO_HOME=${DYNAMO_HOME:-/home/ubuntu/dynamo}
+# Use WORKSPACE_DIR if set (should be set by Dockerfile.local_dev)
+# Otherwise, use /workspace as fallback
+WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}"
 
 # Ensure we're not running as root
 if [ "$(id -u)" -eq 0 ]; then
@@ -53,11 +54,12 @@ show_and_run() {
 set -x
 
 # Pre-commit hooks
-cd $DYNAMO_HOME && pre-commit install && retry pre-commit install-hooks
+cd $WORKSPACE_DIR && pre-commit install && retry pre-commit install-hooks
 pre-commit run --all-files || true # don't fail the build if pre-commit hooks fail
 
-# Set build directory
-export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$DYNAMO_HOME/.build/target}
+# Use CARGO_TARGET_DIR from environment (should be set by Dockerfile.local_dev)
+# Fallback to $WORKSPACE_DIR/target if not set
+export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$WORKSPACE_DIR/target}
 mkdir -p $CARGO_TARGET_DIR
 
 uv pip uninstall --yes ai-dynamo ai-dynamo-runtime 2>/dev/null || true
@@ -78,27 +80,16 @@ cargo build --locked --profile dev --features dynamo-llm/block-manager
 # fi
 
 # install ai-dynamo-runtime
-(cd $DYNAMO_HOME/lib/bindings/python && retry maturin develop)
+(cd $WORKSPACE_DIR/lib/bindings/python && retry maturin develop)
 
 # install ai-dynamo
-cd $DYNAMO_HOME && retry env DYNAMO_BIN_PATH=$CARGO_TARGET_DIR/debug uv pip install -e .
+cd $WORKSPACE_DIR && retry env DYNAMO_BIN_PATH=$CARGO_TARGET_DIR/debug uv pip install -e .
 
 { set +x; } 2>/dev/null
 
 echo -e "\n" >> ~/.bashrc
 echo "# === This section is generated by the post-create.sh script ===" >> ~/.bashrc
 
-# Check if PYTHONPATH is already set in environment
-if [ -n "${PYTHONPATH:-}" ]; then
-    # PYTHONPATH exists, replace /workspace with $DYNAMO_HOME
-    export PYTHONPATH=$(echo "$PYTHONPATH" | sed "s|/workspace|$DYNAMO_HOME|g")
-    # Also add to .bashrc for persistence
-    if ! grep -q "export PYTHONPATH=" ~/.bashrc; then
-        echo "# PYTHONPATH modified from /workspace to use DYNAMO_HOME" >> ~/.bashrc
-        show_and_run echo "export PYTHONPATH=\"$PYTHONPATH\"" >> ~/.bashrc
-    fi
-fi
-
 if ! grep -q "export GPG_TTY=" ~/.bashrc; then
     show_and_run echo 'export GPG_TTY=$(tty)' >> ~/.bashrc
 fi
@@ -123,7 +114,7 @@ else
     echo "⚠️ SSH agent forwarding not configured - SSH_AUTH_SOCK is not set"
 fi
 
-show_and_run $DYNAMO_HOME/deploy/sanity_check.py
+show_and_run $WORKSPACE_DIR/deploy/sanity_check.py
 
 cat <<EOF
 

.devcontainer/sglang/devcontainer.json

@@ -32,7 +32,7 @@
                 // Disable automatic copying of .gitconfig to avoid errors
                 "dev.containers.copyGitConfig": false,
                 "terminal.integrated.defaultProfile.linux": "bash",
-                "terminal.integrated.cwd": "/home/ubuntu/dynamo",
+                "terminal.integrated.cwd": "/workspace",
 
                 "python.defaultInterpreterPath": "/opt/dynamo/venv/bin/python",
                 "python.linting.enabled": true,
@@ -41,7 +41,7 @@
                 "rust-analyzer.checkOnSave.command": "clippy",
                 "rust-analyzer.checkOnSave.enable": true,
                 "rust-analyzer.cargo.buildScripts.enable": true,
-                "rust-analyzer.cargo.targetDir": "/home/ubuntu/dynamo/.build/target",
+                "rust-analyzer.cargo.targetDir": "/workspace/target",
                 "rust-analyzer.procMacro.enable": true,
                 "rust-analyzer.completion.autoimport.enable": true,
 
@@ -59,16 +59,10 @@
             }
         }
     },
-    "workspaceFolder": "/home/ubuntu/dynamo",
-    "workspaceMount": "source=${localWorkspaceFolder},target=/home/ubuntu/dynamo,type=bind,consistency=cached",
+    "workspaceFolder": "/workspace",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
     "userEnvProbe": "interactiveShell",
-    "postCreateCommand": "/bin/bash /home/ubuntu/dynamo/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
-    "containerEnv": {
-        "DYNAMO_HOME": "/home/ubuntu/dynamo",
-        "CARGO_HOME": "/home/ubuntu/dynamo/.build/.cargo",
-        "RUSTUP_HOME": "/home/ubuntu/dynamo/.build/.rustup",
-        "CARGO_TARGET_DIR": "/home/ubuntu/dynamo/.build/target"
-    },
+    "postCreateCommand": "/bin/bash /workspace/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
     "remoteEnv": {
         // Optional convenience tokens passed from host. SSH agent is forwarded via your IDE setting, not here by default.
         "GITHUB_TOKEN": "${localEnv:GITHUB_TOKEN}",

.devcontainer/trtllm/devcontainer.json

@@ -32,7 +32,7 @@
                 // Disable automatic copying of .gitconfig to avoid errors
                 "dev.containers.copyGitConfig": false,
                 "terminal.integrated.defaultProfile.linux": "bash",
-                "terminal.integrated.cwd": "/home/ubuntu/dynamo",
+                "terminal.integrated.cwd": "/workspace",
 
                 "python.defaultInterpreterPath": "/opt/dynamo/venv/bin/python",
                 "python.linting.enabled": true,
@@ -41,7 +41,7 @@
                 "rust-analyzer.checkOnSave.command": "clippy",
                 "rust-analyzer.checkOnSave.enable": true,
                 "rust-analyzer.cargo.buildScripts.enable": true,
-                "rust-analyzer.cargo.targetDir": "/home/ubuntu/dynamo/.build/target",
+                "rust-analyzer.cargo.targetDir": "/workspace/target",
                 "rust-analyzer.procMacro.enable": true,
                 "rust-analyzer.completion.autoimport.enable": true,
 
@@ -59,16 +59,10 @@
             }
         }
     },
-    "workspaceFolder": "/home/ubuntu/dynamo",
-    "workspaceMount": "source=${localWorkspaceFolder},target=/home/ubuntu/dynamo,type=bind,consistency=cached",
+    "workspaceFolder": "/workspace",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
     "userEnvProbe": "interactiveShell",
-    "postCreateCommand": "/bin/bash /home/ubuntu/dynamo/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
-    "containerEnv": {
-        "DYNAMO_HOME": "/home/ubuntu/dynamo",
-        "CARGO_HOME": "/home/ubuntu/dynamo/.build/.cargo",
-        "RUSTUP_HOME": "/home/ubuntu/dynamo/.build/.rustup",
-        "CARGO_TARGET_DIR": "/home/ubuntu/dynamo/.build/target"
-    },
+    "postCreateCommand": "/bin/bash /workspace/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
     "remoteEnv": {
         // Optional convenience tokens passed from host. SSH agent is forwarded via your IDE setting, not here by default.
         "GITHUB_TOKEN": "${localEnv:GITHUB_TOKEN}",

.devcontainer/vllm/devcontainer.json

@@ -32,7 +32,7 @@
                 // Disable automatic copying of .gitconfig to avoid errors
                 "dev.containers.copyGitConfig": false,
                 "terminal.integrated.defaultProfile.linux": "bash",
-                "terminal.integrated.cwd": "/home/ubuntu/dynamo",
+                "terminal.integrated.cwd": "/workspace",
 
                 "python.defaultInterpreterPath": "/opt/dynamo/venv/bin/python",
                 "python.linting.enabled": true,
@@ -41,7 +41,7 @@
                 "rust-analyzer.checkOnSave.command": "clippy",
                 "rust-analyzer.checkOnSave.enable": true,
                 "rust-analyzer.cargo.buildScripts.enable": true,
-                "rust-analyzer.cargo.targetDir": "/home/ubuntu/dynamo/.build/target",
+                "rust-analyzer.cargo.targetDir": "/workspace/target",
                 "rust-analyzer.procMacro.enable": true,
                 "rust-analyzer.completion.autoimport.enable": true,
 
@@ -59,16 +59,10 @@
             }
         }
     },
-    "workspaceFolder": "/home/ubuntu/dynamo",
-    "workspaceMount": "source=${localWorkspaceFolder},target=/home/ubuntu/dynamo,type=bind,consistency=cached",
+    "workspaceFolder": "/workspace",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
     "userEnvProbe": "interactiveShell",
-    "postCreateCommand": "/bin/bash /home/ubuntu/dynamo/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
-    "containerEnv": {
-        "DYNAMO_HOME": "/home/ubuntu/dynamo",
-        "CARGO_HOME": "/home/ubuntu/dynamo/.build/.cargo",
-        "RUSTUP_HOME": "/home/ubuntu/dynamo/.build/.rustup",
-        "CARGO_TARGET_DIR": "/home/ubuntu/dynamo/.build/target"
-    },
+    "postCreateCommand": "/bin/bash /workspace/.devcontainer/post-create.sh", // Runs cargo build and pip installs packages
     "remoteEnv": {
         // Optional convenience tokens passed from host. SSH agent is forwarded via your IDE setting, not here by default.
         "GITHUB_TOKEN": "${localEnv:GITHUB_TOKEN}",

container/Dockerfile.local_dev

@@ -88,6 +88,7 @@ ENV WORKSPACE_DIR=${WORKSPACE_DIR}
 #                * Contains rust toolchain binaries that must be at expected system paths
 #                * Workspace mount point would break rustup's toolchain resolution
 # - PATH: Includes cargo binaries for rust tool access
+ENV HOME=/home/$USERNAME
 ENV DYNAMO_HOME=${WORKSPACE_DIR}
 ENV CARGO_TARGET_DIR=${WORKSPACE_DIR}/target
 ENV CARGO_HOME=${HOME}/.cargo
@@ -110,7 +111,6 @@ RUN rsync -a --chown=$USER_UID:$USER_GID ${VIRTUAL_ENV}/ /tmp/venv-temp/ && \
 
 # At this point, we are executing as the ubuntu user
 USER $USERNAME
-ENV HOME=/home/$USERNAME
 WORKDIR $HOME
 
 # https://code.visualstudio.com/remote/advancedcontainers/persist-bash-history

container/README.md

@@ -43,34 +43,73 @@ These targets are specified with `build.sh --target <target>` and correspond to
 
 Additional targets are available in the Dockerfiles for specific build stages and use cases.
 
+| Feature | **dev + `run.sh`** | **local-dev + `run.sh`** | **local-dev + Dev Container** |
+|---------|-------------------|--------------------------|-------------------------------|
+| **Default User** | root | ubuntu | ubuntu |
+| **User Setup** | None | Matches UID/GID of `build.sh` user | Matches UID/GID of `build.sh` user |
+| **Permissions** | root | ubuntu with sudo | ubuntu with sudo |
+| **Home Directory** | `/root` | `/home/ubuntu` | `/home/ubuntu` |
+| **Working Directory** | `/workspace` | `/workspace` | `/workspace` |
+| **Rust Toolchain** | System install (`/usr/local/rustup`, `/usr/local/cargo`) | User install (`~/.rustup`, `~/.cargo`) | User install (`~/.rustup`, `~/.cargo`) |
+| **Python Env** | root owned | User owned venv | User owned venv |
+| **File Permissions** | root-level | user-level, safe | user-level, safe |
+| **Compatibility** | Legacy workflows, workspace not writable on NFS | workspace writable on NFS | workspace writable on NFS |
+
+## Environment Variables Across Build Stages
+
+Understanding how environment variables change across different build stages is crucial for development and debugging. The Dynamo build system uses a multi-stage Docker build process where environment variables are set, inherited, and overridden at different stages.
+
+### Build Stage Flow
+
 ```
-Feature           │ 1. dev + `run.sh`     │ 2. local-dev + `run.sh`  │ 3. local-dev + Dev Container
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Default User      │ root                  │ ubuntu                   │ ubuntu
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-User Setup        │ None                  │ Matches UID/GID of       │ Matches UID/GID of
-                  │                       │ `build.sh` user          │ `build.sh` user
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Permissions       │ root                  │ ubuntu with sudo         │ ubuntu with sudo
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Home Directory    │ /root                 │ /home/ubuntu             │ /home/ubuntu
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Working Directory │ /workspace            │ /workspace               │ /home/ubuntu/dynamo
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Rust Toolchain    │ System install        │ User install (~/.rustup, │ User install (~/.rustup,
-                  │ (/usr/local/rustup,   │  ~/.cargo)               │  ~/.cargo)
-                  │  /usr/local/cargo)    │                          │
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Python Env        │ root owned            │ User owned venv          │ User owned venv
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-File Permissions  │ root-level            │ user-level, safe         │ user-level, safe
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
-Compatibility     │ Legacy workflows,     │ workspace writable on NFS│workspace writable on NFS
-                  │   workspace not       │                          │
-                  │   writable on NFS     │                          │
-──────────────────┼───────────────────────┼──────────────────────────┼────────────────────────────
+Dockerfile → base → dev (dynamo-base image)
+              ↓
+Dockerfile.vllm → framework → runtime → dev (vllm dev image)
+                                         ↓
+Dockerfile.local_dev → local-dev (from vllm dev image)
 ```
 
+### Environment Variables by Stage
+
+| Variable | **base** | **base→dev** | **vllm→framework** | **vllm→runtime** | **vllm→dev** | **local-dev** |
+|----------|----------|--------------|--------------------|-----------------|--------------|--------------------|
+| **DYNAMO_HOME** | ❌ Not set | `/opt/dynamo` | ❌ Not set | `/opt/dynamo` | `/workspace` ✅ **OVERRIDE** | `/workspace` (inherited) |
+| **WORKSPACE_DIR** | ❌ Not set | ❌ Not set | ❌ Not set | ❌ Not set | `/workspace` | `/workspace` (inherited) |
+| **CARGO_TARGET_DIR** | ❌ Not set | `/opt/dynamo/target` | ❌ Not set | ❌ Not set | `/workspace/target` ✅ **OVERRIDE** | `/workspace/target` (inherited) |
+| **VIRTUAL_ENV** | `/opt/dynamo/venv` | (inherited) | `/opt/dynamo/venv` | `/opt/dynamo/venv` | `/opt/dynamo/venv` ✅ **REDEFINE** | `/opt/dynamo/venv` (inherited) |
+| **RUSTUP_HOME** | `/usr/local/rustup` | (inherited) | ❌ Not set | ❌ Not set | `/usr/local/rustup` | `/home/ubuntu/.rustup` ✅ **OVERRIDE** |
+| **CARGO_HOME** | `/usr/local/cargo` | (inherited) | ❌ Not set | ❌ Not set | `/usr/local/cargo` | `/home/ubuntu/.cargo` ✅ **OVERRIDE** |
+| **USERNAME** | ❌ Not set | ❌ Not set | ❌ Not set | ❌ Not set | ❌ Not set | `ubuntu` ✅ **NEW** |
+| **HOME** | (system default) | (system default) | (system default) | (system default) | (system default) | `/home/ubuntu` ✅ **NEW** |
+| **PATH** | (includes cargo) | (inherited) | (system default) | (includes venv, etcd, ucx) | `/usr/local/cargo/bin:$PATH` | `/home/ubuntu/.cargo/bin:$PATH` ✅ **OVERRIDE** |
+
+### Key Insights
+
+**1. DYNAMO_HOME Dual Purpose:**
+- `base→dev` and `vllm→runtime`: `/opt/dynamo` - For **installed/packaged** Dynamo (CI, production)
+- `vllm→dev` and `local-dev`: `/workspace` - For **development** with source code mounted from host
+
+**2. Rust Toolchain Location:**
+- `dev` target: System-wide at `/usr/local/rustup` and `/usr/local/cargo` (suitable for root)
+- `local-dev` target: User-specific at `/home/ubuntu/.rustup` and `/home/ubuntu/.cargo` (proper UID/GID ownership)
+
+**3. Build Artifacts Location:**
+- `base→dev`: `/opt/dynamo/target` - Build artifacts with installed package
+- `vllm→dev` onward: `/workspace/target` - Build artifacts in mounted workspace for persistence
+
+**4. Variables That Stay Constant:**
+- `VIRTUAL_ENV`: Always `/opt/dynamo/venv` (ownership changes in local-dev via rsync)
+- `WORKSPACE_DIR`: Always `/workspace` once set in vllm→dev
+- `DYNAMO_HOME`: Always `/workspace` once overridden in vllm→dev (for development)
+
+**5. local-dev Specific Changes:**
+From `Dockerfile.local_dev`, the Rust toolchain is moved to user home because:
+- Workspace mount points may change, breaking toolchain paths
+- User needs ownership of cargo binaries and registry for package installation
+- Toolchain requires consistent system paths that don't depend on workspace location
+
+The Python venv ownership is also updated via rsync in local-dev to match the user's UID/GID, ensuring package installation permissions work correctly.
+
 ## Usage Guidelines
 
 - **Use dev + `run.sh`**: for command-line testing. Runs as root user