Skip to content

GitLab

"model/git@developer.sourcefind.cn:OpenDAS/ollama.git" did not exist on "4a48937ef172df630e8bf610a38df0f5a887d243"
Unverified Commit b1f6d89b authored by Yineng Zhang's avatar Yineng Zhang Committed by GitHub
Browse files

fix: update truss bench_serving (#5683)

parent 7c99103f
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
python/sglang/bench_serving.py
View file @ b1f6d89b
...@@ -295,7 +295,7 @@ async def async_request_truss( ...@@ -295,7 +295,7 @@ async def async_request_truss(
# NOTE: Some completion API might have a last # NOTE: Some completion API might have a last
# usage summary response without a token so we # usage summary response without a token so we
# want to check a token was generated # want to check a token was generated
if data["choices"][0]["delta"]["content"]: if data["choices"][0]["text"]:
timestamp = time.perf_counter() timestamp = time.perf_counter()
# First token # First token
if ttft == 0.0: if ttft == 0.0:
...@@ -307,7 +307,7 @@ async def async_request_truss( ...@@ -307,7 +307,7 @@ async def async_request_truss(
output.itl.append(timestamp - most_recent_timestamp) output.itl.append(timestamp - most_recent_timestamp)
most_recent_timestamp = timestamp most_recent_timestamp = timestamp
generated_text += data["choices"][0]["delta"]["content"] generated_text += data["choices"][0]["text"]
output.generated_text = generated_text output.generated_text = generated_text
output.success = True output.success = True
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment