fix: fix examples (#2459)

a4e06895 · julienmancuso · GitHub · ccc8c627 · a4e06895 · a4e06895
Unverified Commit a4e06895 authored Aug 15, 2025 by julienmancuso Committed by GitHub Aug 15, 2025
8 changed files
--- a/components/backends/sglang/deploy/agg.yaml
+++ b/components/backends/sglang/deploy/agg.yaml
@@ -47,16 +47,12 @@ spec:
            - /bin/sh
            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.worker
-            - "dynamo.sglang.worker"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --page-size 16
-            - "--served-model-name"
+              --tp 1
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --trust-remote-code
-            - "--page-size"
+              --skip-tokenizer-init
-            - "16"
-            - "--tp"
-            - "1"
-            - "--trust-remote-code"
-            - "--skip-tokenizer-init"
--- a/components/backends/sglang/deploy/agg_router.yaml
+++ b/components/backends/sglang/deploy/agg_router.yaml
@@ -47,16 +47,11 @@ spec:
            - /bin/sh
            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.worker
-            - "dynamo.sglang.worker"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --page-size 16
-            - "--served-model-name"
+              --tp 1
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --trust-remote-code
-            - "--page-size"
+              --skip-tokenizer-init
-            - "16"
-            - "--tp"
-            - "1"
-            - "--trust-remote-code"
-            - "--skip-tokenizer-init"
--- a/components/backends/sglang/deploy/disagg-multinode.yaml
+++ b/components/backends/sglang/deploy/disagg-multinode.yaml
@@ -48,25 +48,17 @@ spec:
          workingDir: /workspace/components/backends/sglang
          command: ["sh", "-c"]
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.decode_worker
-            - "dynamo.sglang.decode_worker"
+              --model-path meta-llama/Llama-3.3-70B-Instruct
-            - "--model-path"
+              --served-model-name meta-llama/Llama-3.3-70B-Instruct
-            - "meta-llama/Llama-3.3-70B-Instruct"
+              --tp-size 8
-            - "--served-model-name"
+              --trust-remote-code
-            - "meta-llama/Llama-3.3-70B-Instruct"
+              --skip-tokenizer-init
-            - "--tp-size"
+              --disaggregation-mode decode
-            - "8"
+              --disaggregation-transfer-backend nixl
-            - "--trust-remote-code"
+              --disaggregation-bootstrap-port 30001
-            - "--skip-tokenizer-init"
+              --mem-fraction-static 0.82
-            - "--disaggregation-mode"
-            - "decode"
-            - "--disaggregation-transfer-backend"
-            - "nixl"
-            - "--disaggregation-bootstrap-port"
-            - "30001"
-            - "--mem-fraction-static"
-            - "0.82"
    prefill:
      multinode:
        nodeCount: 2
@@ -88,22 +80,14 @@ spec:
          workingDir: /workspace/components/backends/sglang
          command: ["sh", "-c"]
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.worker
-            - "dynamo.sglang.worker"
+              --model-path meta-llama/Llama-3.3-70B-Instruct
-            - "--model-path"
+              --served-model-name meta-llama/Llama-3.3-70B-Instruct
-            - "meta-llama/Llama-3.3-70B-Instruct"
+              --tp-size 8
-            - "--served-model-name"
+              --trust-remote-code
-            - "meta-llama/Llama-3.3-70B-Instruct"
+              --skip-tokenizer-init
-            - "--tp-size"
+              --disaggregation-mode prefill
-            - "8"
+              --disaggregation-transfer-backend nixl
-            - "--trust-remote-code"
+              --disaggregation-bootstrap-port 30001
-            - "--skip-tokenizer-init"
+              --mem-fraction-static 0.82
-            - "--disaggregation-mode"
-            - "prefill"
-            - "--disaggregation-transfer-backend"
-            - "nixl"
-            - "--disaggregation-bootstrap-port"
-            - "30001"
-            - "--mem-fraction-static"
-            - "0.82"
--- a/components/backends/sglang/deploy/disagg.yaml
+++ b/components/backends/sglang/deploy/disagg.yaml
@@ -20,7 +20,7 @@ spec:
          memory: "40Gi"
      extraPodSpec:
        mainContainer:
-          image: nvcr.io/nvidian/nim-llm-dev/sglang-runtime:hzhou-0808-07
+          image: my-registry/sglang-runtime:my-tag
          workingDir: /workspace/components/backends/sglang
          command: ["sh", "-c"]
          args:
@@ -41,29 +41,23 @@ spec:
          gpu: "1"
      extraPodSpec:
        mainContainer:
-          image: nvcr.io/nvidian/nim-llm-dev/sglang-runtime:hzhou-0808-07
+          image: my-registry/sglang-runtime:my-tag
          workingDir: /workspace/components/backends/sglang
          command:
            - /bin/sh
            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.decode_worker
-            - "dynamo.sglang.decode_worker"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --page-size 16
-            - "--served-model-name"
+              --tp 1
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --trust-remote-code
-            - "--page-size"
+              --skip-tokenizer-init
-            - "16"
+              --disaggregation-mode decode
-            - "--tp"
+              --disaggregation-transfer-backend nixl
-            - "1"
-            - "--trust-remote-code"
-            - "--skip-tokenizer-init"
-            - "--disaggregation-mode"
-            - "decode"
-            - "--disaggregation-transfer-backend"
-            - "nixl"
    SGLangPrefillWorker:
      envFromSecret: hf-token-secret
      dynamoNamespace: sglang-disagg
@@ -80,26 +74,19 @@ spec:
          gpu: "1"
      extraPodSpec:
        mainContainer:
-          image: nvcr.io/nvidian/nim-llm-dev/sglang-runtime:hzhou-0808-07
+          image: my-registry/sglang-runtime:my-tag
          workingDir: /workspace/components/backends/sglang
          command:
            - /bin/sh
            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.worker
-            - "dynamo.sglang.worker"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --page-size 16
-            - "--served-model-name"
+              --tp 1
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --trust-remote-code
-            - "--page-size"
+              --skip-tokenizer-init
-            - "16"
+              --disaggregation-mode prefill
-            - "--tp"
+              --disaggregation-transfer-backend nixl
-            - "1"
-            - "--trust-remote-code"
-            - "--skip-tokenizer-init"
-            - "--disaggregation-mode"
-            - "prefill"
-            - "--disaggregation-transfer-backend"
-            - "nixl"
--- a/components/backends/sglang/deploy/disagg_planner.yaml
+++ b/components/backends/sglang/deploy/disagg_planner.yaml
@@ -71,14 +71,16 @@ spec:
        mainContainer:
          image: nvcr.io/nvidian/nim-llm-dev/sglang-runtime:hzhou-0811-1
          workingDir: /workspace/components/planner/src/dynamo/planner
+          command:
+            - /bin/sh
+            - -c
          args:
-            - python
+            - >-
-            - -m
+              python3 -m planner_sla
-            - planner_sla
+              --environment=kubernetes
-            - --environment=kubernetes
+              --backend=sglang
-            - --backend=sglang
+              --adjustment-interval=60
-            - --adjustment-interval=60
+              --profile-results-dir=/workspace/profiling_results
-            - --profile-results-dir=/workspace/profiling_results
    Prometheus: # NOTE: this is set on Prometheus to ensure a service is created for the Prometheus component. This is a workaround and should be managed differently.
      dynamoNamespace: dynamo
      componentType: frontend
@@ -139,24 +141,20 @@ spec:
        mainContainer:
          image: nvcr.io/nvidian/nim-llm-dev/sglang-runtime:hzhou-0811-1
          workingDir: /workspace/components/backends/sglang
+          command:
+            - /bin/sh
+            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.decode_worker
-            - "dynamo.sglang.decode_worker"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --page-size 16
-            - "--served-model-name"
+              --tp 1
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --trust-remote-code
-            - "--page-size"
+              --skip-tokenizer-init
-            - "16"
+              --disaggregation-mode decode
-            - "--tp"
+              --disaggregation-transfer-backend nixl
-            - "1"
-            - "--trust-remote-code"
-            - "--skip-tokenizer-init"
-            - "--disaggregation-mode"
-            - "decode"
-            - "--disaggregation-transfer-backend"
-            - "nixl"
    SGLangPrefillWorker:
      dynamoNamespace: dynamo
      envFromSecret: hf-token-secret
@@ -175,21 +173,17 @@ spec:
        mainContainer:
          image: nvcr.io/nvidian/nim-llm-dev/sglang-runtime:hzhou-0811-1
          workingDir: /workspace/components/backends/sglang
+          command:
+            - /bin/sh
+            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.sglang.worker
-            - "dynamo.sglang.worker"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --page-size 16
-            - "--served-model-name"
+              --tp 1
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --trust-remote-code
-            - "--page-size"
+              --skip-tokenizer-init
-            - "16"
+              --disaggregation-mode prefill
-            - "--tp"
+              --disaggregation-transfer-backend nixl
-            - "1"
-            - "--trust-remote-code"
-            - "--skip-tokenizer-init"
-            - "--disaggregation-mode"
-            - "prefill"
-            - "--disaggregation-transfer-backend"
-            - "nixl"
--- a/components/backends/trtllm/deploy/agg.yaml
+++ b/components/backends/trtllm/deploy/agg.yaml
@@ -45,13 +45,12 @@ spec:
        mainContainer:
          image: nvcr.io/nvidian/nim-llm-dev/trtllm-runtime:dep-233.17
          workingDir: /workspace/components/backends/trtllm
+          command:
+            - /bin/sh
+            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.trtllm
-            - "dynamo.trtllm"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --extra-engine-args engine_configs/agg.yaml
-            - "--served-model-name"
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
-            - "--extra-engine-args"
-            - "engine_configs/agg.yaml"
--- a/components/backends/trtllm/deploy/agg_router.yaml
+++ b/components/backends/trtllm/deploy/agg_router.yaml
@@ -38,14 +38,13 @@ spec:
        mainContainer:
          image: nvcr.io/nvidian/nim-llm-dev/trtllm-runtime:dep-233.17
          workingDir: /workspace/components/backends/trtllm
+          command:
+            - /bin/sh
+            - -c
          args:
-            - "python3"
+            - >-
-            - "-m"
+              python3 -m dynamo.trtllm
-            - "dynamo.trtllm"
+              --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "--model-path"
+              --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+              --extra-engine-args engine_configs/agg.yaml
-            - "--served-model-name"
+              --publish-events-and-metrics
-            - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
-            - "--extra-engine-args"
-            - "engine_configs/agg.yaml"
-            - "--publish-events-and-metrics"
--- a/components/backends/vllm/deploy/disagg_planner.yaml
+++ b/components/backends/vllm/deploy/disagg_planner.yaml
@@ -73,14 +73,16 @@ spec:
        mainContainer:
          image: nvcr.io/nvidian/nim-llm-dev/vllm-runtime:dep-253.17
          workingDir: /workspace/components/planner/src/dynamo/planner
+          command:
+            - /bin/sh
+            - -c
          args:
-            - python
+            - >-
-            - -m
+              python3 -m planner_sla
-            - planner_sla
+              --environment=kubernetes
-            - --environment=kubernetes
+              --backend=vllm
-            - --backend=vllm
+              --adjustment-interval=60
-            - --adjustment-interval=60
+              --profile-results-dir=/workspace/profiling_results
-            - --profile-results-dir=/workspace/profiling_results
    Prometheus: # NOTE: this is set on Prometheus to ensure a service is created for the Prometheus component. This is a workaround and should be managed differently.
      dynamoNamespace: vllm-disagg-planner
      componentType: frontend