setup_benchmarking_resources.sh 3.46 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#!/bin/bash

# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

set -euo pipefail

# Resolve repo root relative to this script
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Inputs
NAMESPACE="${NAMESPACE:-default}"
HF_TOKEN="${HF_TOKEN:-}"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log() { echo -e "${BLUE}[INFO]${NC} $*"; }
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
err() { echo -e "${RED}[ERROR]${NC} $*"; }

usage() {
  cat << EOF
Usage:
  NAMESPACE=<ns> [HF_TOKEN=<token>] deploy/utils/setup_benchmarking_resources.sh

Sets up benchmarking and profiling resources in an existing Dynamo namespace:
33
  - Applies common manifests (PVC)
34
35
36
37
  - Creates HuggingFace token secret if HF_TOKEN provided
  - Installs benchmark dependencies if requirements.txt exists

Prerequisites:
38
  - Dynamo Kubernetes Platform must already be installed in the namespace
39
40
41
42
43
44
45
46
47
48
49
50
  - kubectl must be configured and pointing to the target cluster

Environment variables:
  NAMESPACE         Target Kubernetes namespace (default: default)
  HF_TOKEN          Hugging Face token; if set, a secret named hf-token-secret is created (optional)
EOF
}

if ! command -v kubectl &>/dev/null; then err "kubectl not found"; exit 1; fi

# Check if namespace exists
if ! kubectl get namespace "$NAMESPACE" &>/dev/null; then
51
  err "Namespace $NAMESPACE does not exist. Please create it first or install Dynamo Kubernetes Platform."
52
53
54
55
56
57
  exit 1
fi

# Check if Dynamo platform is installed
if ! kubectl get pods -n "$NAMESPACE" | grep -q "dynamo-platform"; then
  warn "Dynamo platform pods not found in namespace $NAMESPACE"
58
  warn "Please ensure Dynamo Kubernetes Platform is installed first:"
59
  warn "  See: docs/kubernetes/installation_guide.md"
60
61
62
63
64
65
66
67
68
69
70
71
72
  if [[ -z "${FORCE:-}" && -z "${YES:-}" ]]; then
    read -p "Continue anyway? [y/N]: " -r ans
    [[ "$ans" =~ ^[Yy]$ ]] || exit 1
  else
    warn "Continuing due to FORCE/YES set."
  fi
fi

# Apply common manifests
log "Applying benchmarking manifests to namespace $NAMESPACE"
export NAMESPACE  # ensure envsubst can see it
for mf in "$(dirname "$0")/manifests"/*.yaml; do
  if [[ -f "$mf" ]]; then
73
    # Skip pvc-access-pod.yaml as it's created on-demand by users
74
    if [[ "$(basename "$mf")" == "pvc-access-pod.yaml" ]]; then
75
      log "Skipping $mf (created on-demand when accessing PVC)"
76
77
78
      continue
    fi

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    if command -v envsubst >/dev/null 2>&1; then
      envsubst < "$mf" | kubectl -n "$NAMESPACE" apply -f -
    else
      warn "envsubst not found; applying manifest without substitution: $mf"
      kubectl -n "$NAMESPACE" apply -f "$mf"
    fi
  fi
done
ok "Benchmarking manifests applied"

# Optional: Create Hugging Face token secret if HF_TOKEN provided
if [[ -n "$HF_TOKEN" ]]; then
  kubectl create secret generic hf-token-secret \
    --from-literal=HF_TOKEN="$HF_TOKEN" \
    -n "$NAMESPACE" \
    --dry-run=client -o yaml | kubectl apply -f -
  ok "hf-token-secret created/updated"
fi


ok "Benchmarking resource setup complete"

# Verify installation
log "Verifying installation..."
kubectl get pvc dynamo-pvc -n "$NAMESPACE" >/dev/null && ok "PVC dynamo-pvc exists" || err "PVC dynamo-pvc not found"

if [[ -n "$HF_TOKEN" ]]; then
  kubectl get secret hf-token-secret -n "$NAMESPACE" >/dev/null && ok "Secret hf-token-secret exists" || err "Secret hf-token-secret not found"
fi