"examples/backends/vllm/deploy/disagg.yaml" did not exist on "439e977d9c751ef80f1ed72f03078dc408137a74"
agg.sh 1.96 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Setup cleanup trap
cleanup() {
    echo "Cleaning up background processes..."
    kill $DYNAMO_PID 2>/dev/null || true
    wait $DYNAMO_PID 2>/dev/null || true
    echo "Cleanup complete."
}
trap cleanup EXIT INT TERM

14
15
# Default values
MODEL="Qwen/Qwen3-0.6B"
16
ENABLE_OTEL=false
17
18
19

# Parse command line arguments
EXTRA_ARGS=()
20
21
while [[ $# -gt 0 ]]; do
    case $1 in
22
23
24
25
        --model-path)
            MODEL="$2"
            shift 2
            ;;
26
27
28
29
30
31
32
        --enable-otel)
            ENABLE_OTEL=true
            shift
            ;;
        -h|--help)
            echo "Usage: $0 [OPTIONS]"
            echo "Options:"
33
            echo "  --model-path <name>  Specify model (default: $MODEL)"
34
35
36
            echo "  --enable-otel        Enable OpenTelemetry tracing"
            echo "  -h, --help           Show this help message"
            echo ""
37
            echo "Additional SGLang/Dynamo flags can be passed and will be forwarded"
38
39
40
41
            echo "Note: System metrics are enabled by default on port 8081 (worker)"
            exit 0
            ;;
        *)
42
43
            EXTRA_ARGS+=("$1")
            shift
44
45
46
47
48
49
50
51
52
53
            ;;
    esac
done

# Enable tracing if requested
if [ "$ENABLE_OTEL" = true ]; then
    export DYN_LOGGING_JSONL=true
    export OTEL_EXPORT_ENABLED=1
    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://localhost:4317}
fi
54
55

# run ingress
56
# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000)
57
OTEL_SERVICE_NAME=dynamo-frontend \
58
python3 -m dynamo.frontend &
59
60
DYNAMO_PID=$!

61
62
# run worker with metrics enabled
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \
63
python3 -m dynamo.sglang \
64
65
  --model-path "$MODEL" \
  --served-model-name "$MODEL" \
66
67
  --page-size 16 \
  --tp 1 \
68
69
  --trust-remote-code \
  --skip-tokenizer-init \
70
71
  --enable-metrics \
  "${EXTRA_ARGS[@]}"