Skip to content

GitLab

"git@developer.sourcefind.cn:OpenDAS/torchaudio.git" did not exist on "37f2b4f03462cac9b20f28e9a4c5d669b96531ae"
Unverified Commit 2e8fc0d4 authored by Simon Mo's avatar Simon Mo Committed by GitHub
Browse files

Fix completion API echo and logprob combo (#1992)

parent dacaf5a4
Hide whitespace changes
Inline Side-by-side
Showing with 9 additions and 4 deletions
vllm/entrypoints/openai/api_server.py
View file @ 2e8fc0d4
...@@ -567,17 +567,22 @@ async def create_completion(request: CompletionRequest, raw_request: Request): ...@@ -567,17 +567,22 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
i = output.index i = output.index
delta_text = output.text[len(previous_texts[i]):] delta_text = output.text[len(previous_texts[i]):]
token_ids = output.token_ids[previous_num_tokens[i]:] token_ids = output.token_ids[previous_num_tokens[i]:]
top_logprobs = output.logprobs[previous_num_tokens[i]:] if request.logprobs is not None:
top_logprobs = output.logprobs[previous_num_tokens[i]:]
else:
top_logprobs = None
offsets = len(previous_texts[i]) offsets = len(previous_texts[i])
if request.echo and not has_echoed[i]: if request.echo and not has_echoed[i]:
if not echo_without_generation: if not echo_without_generation:
delta_text = res.prompt + delta_text delta_text = res.prompt + delta_text
token_ids = res.prompt_token_ids + token_ids token_ids = res.prompt_token_ids + token_ids
top_logprobs = res.prompt_logprobs + top_logprobs if top_logprobs:
else: top_logprobs = res.prompt_logprobs + top_logprobs
else: # only just return the prompt
delta_text = res.prompt delta_text = res.prompt
token_ids = res.prompt_token_ids token_ids = res.prompt_token_ids
top_logprobs = res.prompt_logprobs if top_logprobs:
top_logprobs = res.prompt_logprobs
has_echoed[i] = True has_echoed[i] = True
if request.logprobs is not None: if request.logprobs is not None:
logprobs = create_logprobs( logprobs = create_logprobs(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment