Skip to content

GitLab

"examples/community/test_tensorrt_controlnet.py" did not exist on "20e92586c1fda968ea3343ba0f44f2b21f3c09d2"
Unverified Commit b1f6d89b authored by Yineng Zhang's avatar Yineng Zhang Committed by GitHub
Browse files

fix: update truss bench_serving (#5683)

parent 7c99103f
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
python/sglang/bench_serving.py
View file @ b1f6d89b
...@@ -295,7 +295,7 @@ async def async_request_truss( ...@@ -295,7 +295,7 @@ async def async_request_truss(
# NOTE: Some completion API might have a last # NOTE: Some completion API might have a last
# usage summary response without a token so we # usage summary response without a token so we
# want to check a token was generated # want to check a token was generated
if data["choices"][0]["delta"]["content"]: if data["choices"][0]["text"]:
timestamp = time.perf_counter() timestamp = time.perf_counter()
# First token # First token
if ttft == 0.0: if ttft == 0.0:
...@@ -307,7 +307,7 @@ async def async_request_truss( ...@@ -307,7 +307,7 @@ async def async_request_truss(
output.itl.append(timestamp - most_recent_timestamp) output.itl.append(timestamp - most_recent_timestamp)
most_recent_timestamp = timestamp most_recent_timestamp = timestamp
generated_text += data["choices"][0]["delta"]["content"] generated_text += data["choices"][0]["text"]
output.generated_text = generated_text output.generated_text = generated_text
output.success = True output.success = True
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment