Skip to content

GitLab

Unverified Commit a4851cfe authored by Chauncey's avatar Chauncey Committed by GitHub
Browse files

[Bugfix]: Fix messy code when using logprobs (#20910) 


Signed-off-by: default avatarchaunceyjiang <chaunceyjiang@gmail.com>
parent 9887e8ec
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 5 deletions
vllm/transformers_utils/detokenizer_utils.py
View file @ a4851cfe
...@@ -78,7 +78,6 @@ def convert_prompt_ids_to_tokens( ...@@ -78,7 +78,6 @@ def convert_prompt_ids_to_tokens(
def convert_ids_list_to_tokens( def convert_ids_list_to_tokens(
tokenizer: AnyTokenizer, tokenizer: AnyTokenizer,
token_ids: list[int], token_ids: list[int],
skip_special_tokens: bool = False,
) -> list[str]: ) -> list[str]:
"""Detokenize the input ids individually. """Detokenize the input ids individually.
...@@ -92,10 +91,8 @@ def convert_ids_list_to_tokens( ...@@ -92,10 +91,8 @@ def convert_ids_list_to_tokens(
""" """
token_str_lst = [] token_str_lst = []
for token_id in token_ids: for token_id in token_ids:
token_str = tokenizer.decode( # use default skip_special_tokens.
[token_id], token_str = tokenizer.decode([token_id])
skip_special_tokens=skip_special_tokens,
)
if token_str is None: if token_str is None:
token_str = "" token_str = ""
token_str_lst.append(token_str) token_str_lst.append(token_str)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment