chore: Add make targets for tooling binaries in operator (#7609)

Signed-off-by: Jont828 <jt572@cornell.edu>

deploy/operator/Makefile

@@ -7,11 +7,16 @@ ENVTEST_K8S_VERSION = 1.29.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
-GOBIN=$(shell go env GOPATH)/bin
+GOBIN := $(shell go env GOPATH)/bin
 else
-GOBIN=$(shell go env GOBIN)
+GOBIN := $(shell go env GOBIN)
 endif
 
+# Platform detection for downloading pre-built binaries.
+GOOS := $(shell go env GOOS)
+GOARCH := $(shell go env GOARCH)
+CURL_RETRIES ?= 3
+
 # CONTAINER_TOOL defines the container tool to be used for building images.
 # Be aware that the target commands are only tested with Docker which is
 # scaffolded by default. However, you might want to replace it to use other
@@ -58,38 +63,34 @@ check: generate manifests generate-api-docs generate-helm-docs
 	@echo "> Check complete"
 
 
-.PHONY: ensure-yq
-ensure-yq:
-	@if ! command -v yq &> /dev/null; then \
-		echo "Installing yq..."; \
-		ARCH=$$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/'); \
-		OS=$$(uname -s | tr '[:upper:]' '[:lower:]'); \
-		wget https://github.com/mikefarah/yq/releases/latest/download/yq_$${OS}_$${ARCH} -O /usr/local/bin/yq && \
-		chmod +x /usr/local/bin/yq; \
-	else \
-		echo "yq is already installed: $$(yq --version)"; \
+.PHONY: tilt-up
+tilt-up: kubectl helm ## Start Tilt for local operator development.
+	@if [ ! -f tilt-settings.yaml ]; then \
+		echo "WARNING: tilt-settings.yaml not found — using defaults."; \
 	fi
+	tilt up
+
 
 .PHONY: manifests
-manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	# Use a large maxDescLen to ensure all field comments are included as OpenAPI descriptions
 	# allowDangerousTypes=true is needed for the EndpointPickerConfig from gateway-api-inference-extension which contains float fields
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd:maxDescLen=100000,allowDangerousTypes=true webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 	echo "Removing name from mainContainer required fields"
 	for file in config/crd/bases/*.yaml; do \
-		yq eval '(.. | select(has("mainContainer")) | .mainContainer.required) |= (. - ["name"])' -i --indent 2 $$file || exit 1; \
+		$(YQ) eval '(.. | select(has("mainContainer")) | .mainContainer.required) |= (. - ["name"])' -i --indent 2 $$file || exit 1; \
 	done
 	echo "Removing containers from extraPodSpec required fields"
 	for file in config/crd/bases/*.yaml; do \
-		yq eval '(.. | select(has("extraPodSpec")) | .extraPodSpec.required) |= (. - ["containers"])' -i --indent 2 $$file || exit 1; \
+		$(YQ) eval '(.. | select(has("extraPodSpec")) | .extraPodSpec.required) |= (. - ["containers"])' -i --indent 2 $$file || exit 1; \
 	done
 	echo "Fixing PluginSpec parameters field: json.RawMessage needs x-kubernetes-preserve-unknown-fields instead of type: string"
 	for file in config/crd/bases/*.yaml; do \
-		yq eval '(.. | select(has("parameters")) | .parameters | select(has("format") and .format == "byte")) |= (del(.format) | del(.type) | .x-kubernetes-preserve-unknown-fields = true)' -i --indent 2 $$file || exit 1; \
+		$(YQ) eval '(.. | select(has("parameters")) | .parameters | select(has("format") and .format == "byte")) |= (del(.format) | del(.type) | .x-kubernetes-preserve-unknown-fields = true)' -i --indent 2 $$file || exit 1; \
 	done
 	echo "Fixing profilingJob template metadata: controller-gen emits bare type: object for metav1.ObjectMeta, add x-kubernetes-preserve-unknown-fields so labels/annotations are accepted"
 	for file in config/crd/bases/*.yaml; do \
-		yq eval '(.. | select(has("profilingJob")) | .profilingJob.properties.template.properties.metadata)."x-kubernetes-preserve-unknown-fields" = true' -i --indent 2 $$file || exit 1; \
+		$(YQ) eval '(.. | select(has("profilingJob")) | .profilingJob.properties.template.properties.metadata)."x-kubernetes-preserve-unknown-fields" = true' -i --indent 2 $$file || exit 1; \
 	done
 	echo "Adding NVIDIA header to CRD files"
 	for file in config/crd/bases/*.yaml; do \
@@ -104,8 +105,8 @@ manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRol
 	done
 	echo "Adding helm.sh/resource-policy: keep annotation to CRD files"
 	for file in config/crd/bases/*.yaml; do \
-		if ! yq eval '.metadata.annotations."helm.sh/resource-policy"' "$$file" | grep -q "keep"; then \
-			yq eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \
+		if ! $(YQ) eval '.metadata.annotations."helm.sh/resource-policy"' "$$file" | grep -q "keep"; then \
+			$(YQ) eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \
 		fi; \
 	done
 	echo "Copying CRD files to operator Helm chart crds/ directory"
@@ -163,7 +164,7 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 ##@ Build
 
 .PHONY: build
-build: manifests generate fmt vet helm ## Build manager binary.
+build: manifests generate fmt vet helmify-chart ## Build manager binary.
 	go build -o bin/manager cmd/main.go
 
 .PHONY: run
@@ -215,42 +216,214 @@ ifndef ignore-not-found
 endif
 
 .PHONY: install
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+install: manifests kustomize kubectl ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
 
 .PHONY: uninstall
-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+uninstall: manifests kustomize kubectl ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
 .PHONY: deploy
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+deploy: manifests kustomize kubectl ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
 .PHONY: undeploy
-undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+undeploy: kustomize kubectl ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
+GPU_OPERATOR_NAMESPACE ?= gpu-operator
+GPU_OPERATOR_EXTRA_HELM_ARGS ?=
+
+.PHONY: install-gpu-operator
+install-gpu-operator: helm ## Install NVIDIA GPU Operator if not already present.
+	@if $(HELM) list -n $(GPU_OPERATOR_NAMESPACE) -q 2>/dev/null | grep -q '^gpu-operator$$'; then \
+		echo "GPU Operator is already installed — skipping."; \
+	else \
+		echo "Installing NVIDIA GPU Operator..."; \
+		$(HELM) repo add nvidia https://helm.ngc.nvidia.com/nvidia --force-update; \
+		$(HELM) repo update nvidia; \
+		$(HELM) upgrade --install gpu-operator nvidia/gpu-operator \
+			--namespace $(GPU_OPERATOR_NAMESPACE) --create-namespace \
+			$(GPU_OPERATOR_EXTRA_HELM_ARGS) \
+			--wait --timeout=600s; \
+		echo "GPU Operator installed successfully."; \
+	fi
+
+.PHONY: install-gpu-operator-no-driver
+install-gpu-operator-no-driver: ## Install NVIDIA GPU Operator, skipping driver install (for AKS, etc. that provide their own).
+	$(MAKE) install-gpu-operator GPU_OPERATOR_EXTRA_HELM_ARGS="--set driver.enabled=false --set toolkit.enabled=false --set operator.runtimeClass=nvidia-container-runtime"
+
+##@ Cleanup
+
+# Namespace and Helm release used by Tilt dev-loop (matches Tiltfile defaults).
+DYNAMO_NAMESPACE ?= dynamo-system
+DYNAMO_HELM_RELEASE ?= dynamo
+# Grace period (seconds) before force-deleting stuck resources.
+TEARDOWN_GRACE_PERIOD ?= 30
+
+.PHONY: tilt-down
+tilt-down: ## Stop Tilt and remove all Tilt-managed resources.
+	@echo "Stopping Tilt..."
+	@tilt down 2>/dev/null || echo "Tilt was not running — skipping."
+
+.PHONY: clean-tilt
+clean-tilt: tilt-down ## Stop Tilt, remove its build artifacts, and delete the dev namespace.
+	@echo "Removing Tilt build artifacts..."
+	@rm -rf tilt_bin
+	@echo "Deleting Tilt dev namespace $(DYNAMO_NAMESPACE)..."
+	@$(KUBECTL) delete namespace $(DYNAMO_NAMESPACE) --ignore-not-found --timeout=$(TEARDOWN_GRACE_PERIOD)s 2>/dev/null || { \
+		echo "WARNING: namespace $(DYNAMO_NAMESPACE) stuck terminating — force-removing finalizers..."; \
+		$(KUBECTL) get namespace $(DYNAMO_NAMESPACE) -o json \
+			| sed 's/"kubernetes"//' \
+			| $(KUBECTL) replace --raw "/api/v1/namespaces/$(DYNAMO_NAMESPACE)/finalize" -f - 2>/dev/null || true; \
+	}
+
+.PHONY: clean-dynamo-crds
+clean-dynamo-crds: kubectl ## Delete all Dynamo CRDs (these survive helm uninstall due to resource-policy: keep).
+	@echo "Deleting Dynamo custom resources..."
+	@for crd in $$($(KUBECTL) get crd -o name 2>/dev/null | grep 'nvidia.com' | sed 's|customresourcedefinition.apiextensions.k8s.io/||'); do \
+		echo "  Deleting all $$crd instances..."; \
+		$(KUBECTL) delete "$$crd" --all -A --timeout=$(TEARDOWN_GRACE_PERIOD)s 2>/dev/null || { \
+			echo "  Timed out — stripping finalizers from remaining $$crd instances..."; \
+			$(KUBECTL) get "$$crd" -A -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null | \
+			while read -r ns name; do \
+				[ -z "$$name" ] && continue; \
+				$(KUBECTL) patch "$$crd" "$$name" -n "$$ns" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true; \
+			done; \
+			$(KUBECTL) delete "$$crd" --all -A --force --grace-period=0 2>/dev/null || true; \
+		}; \
+	done
+	@echo "Deleting Dynamo CRDs..."
+	@$(KUBECTL) get crd -o name 2>/dev/null | grep 'nvidia.com' | xargs -r $(KUBECTL) delete --timeout=$(TEARDOWN_GRACE_PERIOD)s 2>/dev/null || \
+		$(KUBECTL) get crd -o name 2>/dev/null | grep 'nvidia.com' | xargs -r $(KUBECTL) delete --force --grace-period=0 2>/dev/null || \
+		echo "No Dynamo CRDs found or already removed."
+
+.PHONY: clean-dynamo-helm
+clean-dynamo-helm: helm ## Uninstall Dynamo Helm releases and delete their namespaces.
+	@echo "Uninstalling Dynamo Helm releases..."
+	@$(HELM) uninstall $(DYNAMO_HELM_RELEASE) --namespace $(DYNAMO_NAMESPACE) --wait --timeout=120s 2>/dev/null || \
+		$(HELM) uninstall $(DYNAMO_HELM_RELEASE) --namespace $(DYNAMO_NAMESPACE) --no-hooks 2>/dev/null || \
+		echo "No Helm release '$(DYNAMO_HELM_RELEASE)' in namespace '$(DYNAMO_NAMESPACE)' — skipped."
+	@$(HELM) uninstall dynamo-platform --namespace dynamo --wait --timeout=120s 2>/dev/null || \
+		$(HELM) uninstall dynamo-platform --namespace dynamo --no-hooks 2>/dev/null || \
+		echo "No Helm release 'dynamo-platform' in namespace 'dynamo' — skipped."
+	@echo "Deleting Dynamo namespaces..."
+	@for ns in $(DYNAMO_NAMESPACE) dynamo; do \
+		if $(KUBECTL) get namespace "$$ns" >/dev/null 2>&1; then \
+			echo "  Deleting namespace $$ns..."; \
+			$(KUBECTL) delete namespace "$$ns" --timeout=$(TEARDOWN_GRACE_PERIOD)s 2>/dev/null || { \
+				echo "  WARNING: namespace $$ns stuck terminating — force-removing finalizers..."; \
+				$(KUBECTL) get namespace "$$ns" -o json \
+					| sed 's/"kubernetes"//' \
+					| $(KUBECTL) replace --raw "/api/v1/namespaces/$$ns/finalize" -f - 2>/dev/null || true; \
+			}; \
+		fi; \
+	done
+
+.PHONY: clean-gpu-operator
+clean-gpu-operator: helm ## Uninstall the NVIDIA GPU Operator and delete its namespace.
+	@if $(HELM) list -n $(GPU_OPERATOR_NAMESPACE) -q 2>/dev/null | grep -q '^gpu-operator$$'; then \
+		echo "Uninstalling GPU Operator..."; \
+		$(HELM) uninstall gpu-operator --namespace $(GPU_OPERATOR_NAMESPACE) --wait --timeout=120s 2>/dev/null || \
+			$(HELM) uninstall gpu-operator --namespace $(GPU_OPERATOR_NAMESPACE) --no-hooks; \
+		echo "Deleting GPU Operator namespace $(GPU_OPERATOR_NAMESPACE)..."; \
+		$(KUBECTL) delete namespace $(GPU_OPERATOR_NAMESPACE) --ignore-not-found --timeout=$(TEARDOWN_GRACE_PERIOD)s 2>/dev/null || { \
+			echo "WARNING: namespace $(GPU_OPERATOR_NAMESPACE) stuck terminating — force-removing finalizers..."; \
+			$(KUBECTL) get namespace $(GPU_OPERATOR_NAMESPACE) -o json \
+				| sed 's/"kubernetes"//' \
+				| $(KUBECTL) replace --raw "/api/v1/namespaces/$(GPU_OPERATOR_NAMESPACE)/finalize" -f - 2>/dev/null || true; \
+		}; \
+		echo "GPU Operator removed."; \
+	else \
+		echo "GPU Operator is not installed — skipping."; \
+	fi
+
+.PHONY: clean-cluster
+clean-cluster: kubectl helm ## Remove ALL Dynamo resources from the cluster and local build artifacts.
+	@echo ""
+	@echo "================================================================"
+	@echo "  Dynamo Full Cluster Cleanup"
+	@echo "================================================================"
+	@echo ""
+	$(MAKE) clean-dynamo-crds
+	$(MAKE) clean-tilt
+	$(MAKE) clean-dynamo-helm
+	$(MAKE) clean-gpu-operator
+	$(MAKE) clean-bin
+	@echo ""
+	@echo "================================================================"
+	@echo "  Cluster is clean."
+	@echo "================================================================"
+	@echo ""
+
 ##@ Dependencies
 
 ## Location to install dependencies to
-LOCALBIN ?= $(shell pwd)/bin
+LOCALBIN ?= $(CURDIR)/bin
 $(LOCALBIN):
 	mkdir -p $(LOCALBIN)
 
-## Tool Binaries
-KUBECTL ?= kubectl
-KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION)
-CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION)
-ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION)
-GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION)
+.PHONY: clean-bin
+clean-bin: ## Remove all downloaded tool binaries.
+	rm -rf $(LOCALBIN)
+
+## Prefer project-local binaries over system-wide ones.
+export PATH := $(LOCALBIN):$(PATH)
 
 ## Tool Versions
+KUBECTL_VERSION ?= v1.31.4
+HELM_VERSION ?= v3.17.3
+YQ_VERSION ?= v4.45.4
 KUSTOMIZE_VERSION ?= v5.5.0
 CONTROLLER_TOOLS_VERSION ?= v0.16.4
 ENVTEST_VERSION ?= release-0.19
 GOLANGCI_LINT_VERSION ?= v1.64.8
 
+## Tool Binaries
+KUBECTL ?= $(LOCALBIN)/kubectl-$(KUBECTL_VERSION)
+HELM ?= $(LOCALBIN)/helm-$(HELM_VERSION)
+YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION)
+KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION)
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION)
+ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION)
+GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION)
+
+## ---------- Binary download targets ----------
+
+.PHONY: kubectl
+kubectl: $(KUBECTL) ## Download kubectl locally if necessary.
+$(KUBECTL):
+	mkdir -p $(LOCALBIN)
+	rm -f $(LOCALBIN)/kubectl-*
+	curl --retry $(CURL_RETRIES) -fsL \
+		https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/$(GOOS)/$(GOARCH)/kubectl \
+		-o $(KUBECTL)
+	ln -sf $(KUBECTL) $(LOCALBIN)/kubectl
+	chmod +x $(KUBECTL) $(LOCALBIN)/kubectl
+
+.PHONY: helm
+helm: $(HELM) ## Download helm locally if necessary.
+$(HELM):
+	mkdir -p $(LOCALBIN)
+	rm -f $(LOCALBIN)/helm-*
+	curl --retry $(CURL_RETRIES) -fsSL https://get.helm.sh/helm-$(HELM_VERSION)-$(GOOS)-$(GOARCH).tar.gz \
+		| tar xzf - --strip-components=1 -C $(LOCALBIN) $(GOOS)-$(GOARCH)/helm
+	mv $(LOCALBIN)/helm $(HELM)
+	ln -sf $(HELM) $(LOCALBIN)/helm
+	chmod +x $(HELM) $(LOCALBIN)/helm
+
+.PHONY: yq
+yq: $(YQ) ## Download yq locally if necessary.
+$(YQ):
+	mkdir -p $(LOCALBIN)
+	rm -f $(LOCALBIN)/yq-*
+	curl --retry $(CURL_RETRIES) -fsL \
+		https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(GOOS)_$(GOARCH) \
+		-o $(YQ)
+	ln -sf $(YQ) $(LOCALBIN)/yq
+	chmod +x $(YQ) $(LOCALBIN)/yq
+
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
 $(KUSTOMIZE): $(LOCALBIN)
@@ -293,7 +466,7 @@ helmify: $(HELMIFY) ## Download helmify locally if necessary.
 $(HELMIFY): $(LOCALBIN)
 	test -s $(LOCALBIN)/helmify || GOBIN=$(LOCALBIN) go install github.com/arttor/helmify/cmd/helmify@v0.4.16
 
-helm: manifests kustomize helmify
+helmify-chart: manifests kustomize helmify
 	$(KUSTOMIZE) build config/default | $(HELMIFY) -image-pull-secrets charts/dynamo-kubernetes-operator
 
 ######################### CRD Reference Docs

deploy/operator/Tiltfile

@@ -17,16 +17,18 @@
 #   4. On code change Tilt recompiles the binary and live-updates it into the
 #      running container — no full image rebuild needed.
 #
-# Prerequisites (must be in PATH):
+# Prerequisites:
 #   - Go >= 1.25        — compiles the manager binary locally
 #   - tilt              — live-reload orchestrator (https://docs.tilt.dev/install.html)
-#   - helm v3           — renders the platform Helm chart
-#   - kubectl >= 1.29   — applies CRDs and creates the namespace
 #   - docker            — builds the live-update container image
 #   - A Kubernetes cluster reachable via your current kubeconfig context
 #
+# Auto-downloaded into ./bin/ on first run (via `make kubectl helm`):
+#   - helm v3           — renders the platform Helm chart
+#   - kubectl >= 1.29   — applies CRDs and creates the namespace
+#
 # Conditional (only when skip_codegen is false, the default):
-#   - yq v4+            — post-processes generated CRD YAML (run `make ensure-yq`)
+#   - yq v4+            — post-processes generated CRD YAML (run `make yq`)
 #   - python3 + pydantic — generates Pydantic models from Go types
 #
 # The tilt restart_process extension is auto-fetched on first `tilt up`.
@@ -99,6 +101,18 @@ OPERATOR_DIR = os.getcwd()                                     # deploy/operator
 HELM_CHART   = os.path.join(OPERATOR_DIR, '..', 'helm', 'charts', 'platform')  # deploy/helm/charts/platform
 CRD_DIR      = os.path.join(HELM_CHART, 'components', 'operator', 'crds')
 
+# ---------------------------------------------------------------------------
+# Local tool binaries — ensure kubectl and helm are downloaded via `make`
+# so that all invocations use the pinned, version-suffixed copies in ./bin/.
+# The Makefile download targets create unversioned symlinks (e.g. bin/kubectl)
+# so we can reference them by their short names here.
+# ---------------------------------------------------------------------------
+TOOLS_BIN_DIR = os.path.join(OPERATOR_DIR, 'bin')
+local('make --directory="%s" kubectl helm' % OPERATOR_DIR, quiet=True)
+
+kubectl_cmd = os.path.join(TOOLS_BIN_DIR, 'kubectl')
+helm_cmd    = os.path.join(TOOLS_BIN_DIR, 'helm')
+
 IMG_NAME = 'controller'
 IMG_TAG  = 'tilt-dev'
 IMG      = (REGISTRY + '/' + IMG_NAME) if REGISTRY else IMG_NAME
@@ -128,7 +142,7 @@ local_resource(
 # CRDs — regenerate & apply via server-side apply on change
 # ---------------------------------------------------------------------------
 SKIP_CODEGEN = settings['skip_codegen']
-_crd_cmd = 'kubectl apply --server-side --force-conflicts -f ' + CRD_DIR
+_crd_cmd = kubectl_cmd + ' apply --server-side --force-conflicts -f ' + CRD_DIR
 if not SKIP_CODEGEN:
     _crd_cmd = 'make generate && make manifests && ' + _crd_cmd
 
@@ -156,8 +170,8 @@ local_resource(
 def render_helm():
     """Render the platform Helm chart with only the operator subchart enabled."""
 
-    helm_cmd = [
-        'helm', 'template', 'dynamo', HELM_CHART,
+    helm_template_cmd = [
+        helm_cmd, 'template', 'dynamo', HELM_CHART,
         '--namespace', NAMESPACE,
         '--set', 'dynamo-operator.enabled=true',
         # Subcharts — NATS is on by default (workers need it)
@@ -177,13 +191,13 @@ def render_helm():
 
     # Wire in imagePullSecrets when a pull secret is configured
     if IMAGE_PULL_SECRET:
-        helm_cmd += ['--set', 'dynamo-operator.imagePullSecrets[0].name=' + IMAGE_PULL_SECRET]
+        helm_template_cmd += ['--set', 'dynamo-operator.imagePullSecrets[0].name=' + IMAGE_PULL_SECRET]
 
     # Append user-provided Helm overrides from tilt-settings
     for k, v in HELM_VALUES.items():
-        helm_cmd += ['--set', '%s=%s' % (k, v)]
+        helm_template_cmd += ['--set', '%s=%s' % (k, v)]
 
-    data = local(helm_cmd, quiet=True)
+    data = local(helm_template_cmd, quiet=True)
 
     # Decode the YAML stream so we can patch individual documents
     decoded = decode_yaml_stream(data)
@@ -217,7 +231,7 @@ def render_helm():
     return encode_yaml_stream(patched)
 
 # Create the namespace before applying anything else
-local('kubectl create namespace %s || true' % NAMESPACE, quiet=True)
+local(kubectl_cmd + ' create namespace %s || true' % NAMESPACE, quiet=True)
 
 k8s_yaml(render_helm())