Add sglang.bench_latency to CI (#1243)

6c498313 · Lianmin Zheng · GitHub · f25f4dfd · 6c498313
Unverified Commit 6c498313 authored Aug 28, 2024 by Lianmin Zheng Committed by GitHub Aug 28, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 5 additions and 0 deletions

.github/workflows/e2e-test.yml .github/workflows/e2e-test.yml +5 -0

No files found.
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -38,6 +38,11 @@ jobs:
          cd test/srt
          python3 -m unittest test_serving_throughput.TestServingThroughput.test_default

+      - name: Benchmark Serving Latency
+        timeout-minutes: 10
+        run: |
+          python3 -m sglang.bench_latency --model meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 1 --input 128 --output 8
+
      - name: Benchmark Serving Throughput (w/o RadixAttention)
        timeout-minutes: 10
        run: |