Skip to content

GitLab

Unverified Commit ccb58b23 authored by Roger Wang's avatar Roger Wang Committed by GitHub
Browse files

[Misc] Fix Benchmark TTFT Calculation for Chat Completions (#3768)

parent 49782fcb
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 3 deletions
benchmarks/backend_request_func.py
View file @ ccb58b23
...@@ -334,7 +334,8 @@ async def async_request_openai_chat_completions( ...@@ -334,7 +334,8 @@ async def async_request_openai_chat_completions(
timestamp = time.perf_counter() timestamp = time.perf_counter()
data = json.loads(chunk) data = json.loads(chunk)
if "content" in data["choices"][0]["delta"]: delta = data["choices"][0]["delta"]
if delta.get("content", None):
# First token # First token
if ttft == 0: if ttft == 0:
ttft = time.perf_counter() - st ttft = time.perf_counter() - st
...@@ -345,8 +346,7 @@ async def async_request_openai_chat_completions( ...@@ -345,8 +346,7 @@ async def async_request_openai_chat_completions(
output.itl.append(timestamp - output.itl.append(timestamp -
most_recent_timestamp) most_recent_timestamp)
generated_text += data["choices"][0]["delta"][ generated_text += delta["content"]
"content"]
most_recent_timestamp = timestamp most_recent_timestamp = timestamp
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment