Skip to content

GitLab

Unverified Commit 65e03893 authored by Nicolò Lucchesi's avatar Nicolò Lucchesi Committed by GitHub
Browse files

[Frontend] Skip unnecessary detokenization when token_id is requested (#24236) 


Signed-off-by: default avatarNickLucche <nlucches@redhat.com>
parent 886ccbe5
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 1 deletion
vllm/entrypoints/openai/serving_chat.py
View file @ 65e03893
...@@ -1419,9 +1419,10 @@ class OpenAIServingChat(OpenAIServing): ...@@ -1419,9 +1419,10 @@ class OpenAIServingChat(OpenAIServing):
step_top_logprobs = top_logprobs[i] step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get( if step_top_logprobs is None or step_top_logprobs.get(
token_id) is None: token_id) is None:
token = tokenizer.decode(token_id)
if should_return_as_token_id: if should_return_as_token_id:
token = f"token_id:{token_id}" token = f"token_id:{token_id}"
else:
token = tokenizer.decode(token_id)
logprobs_content.append( logprobs_content.append(
ChatCompletionLogProbsContent( ChatCompletionLogProbsContent(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment