Unverified Commit b1f6d89b authored by Yineng Zhang's avatar Yineng Zhang Committed by GitHub
Browse files

fix: update truss bench_serving (#5683)

parent 7c99103f
......@@ -295,7 +295,7 @@ async def async_request_truss(
# NOTE: Some completion API might have a last
# usage summary response without a token so we
# want to check a token was generated
if data["choices"][0]["delta"]["content"]:
if data["choices"][0]["text"]:
timestamp = time.perf_counter()
# First token
if ttft == 0.0:
......@@ -307,7 +307,7 @@ async def async_request_truss(
output.itl.append(timestamp - most_recent_timestamp)
most_recent_timestamp = timestamp
generated_text += data["choices"][0]["delta"]["content"]
generated_text += data["choices"][0]["text"]
output.generated_text = generated_text
output.success = True
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment