Skip to content

GitLab

"docs/backends/trtllm/multinode" did not exist on "39d645e58647d6adb074650e46be5de25f3f3bc6"
Unverified Commit 8904fc4d authored by haosdent's avatar haosdent Committed by GitHub
Browse files

[Bugfix] Fix V1 logprobs empty strings for multi-byte UTF-8 tokens when logprobs > 0 (#34875) 


Signed-off-by: default avatarhaosdent <haosdent@gmail.com>
parent 1a2c1763
Show whitespace changes
Inline Side-by-side
Showing with 348 additions and 196 deletions
tests/v1/sample/test_logprobs.py
View file @ 8904fc4d
......@@ -539,6 +539,10 @@ class TestCorrectDecodedToken:
result in the Unicode replacement character "�" (U+FFFD). This commonly
happens with byte-fallback tokenization when multi-byte UTF-8 characters
are split across tokens.
The method signature is _correct_decoded_token(token_id, context_token_ids)
where token_id is the single token to correct and context_token_ids are
the preceding sampled tokens in sequential order.
"""
@pytest.fixture
......@@ -550,8 +554,8 @@ class TestCorrectDecodedToken:
return tokenizer
@pytest.fixture
def processor_with_empty_logprobs(self, mock_tokenizer):
"""Create a LogprobsProcessor with empty logprobs."""
def processor(self, mock_tokenizer):
"""Create a LogprobsProcessor."""
from vllm.v1.engine.logprobs import LogprobsProcessor
processor = LogprobsProcessor(
......@@ -564,209 +568,191 @@ class TestCorrectDecodedToken:
)
return processor
@pytest.fixture
def processor_with_previous_logprobs(self, mock_tokenizer):
"""Create a LogprobsProcessor with previous logprobs."""
from vllm.v1.engine.logprobs import LogprobsProcessor
def test_correction_with_context(self, processor):
"""Test correction using context from preceding sampled tokens.
processor = LogprobsProcessor(
tokenizer=mock_tokenizer,
logprobs=[{123: None}], # Previous token ID is 123
prompt_logprobs=None,
cumulative_logprob=0.0,
num_logprobs=1,
num_prompt_logprobs=None,
)
return processor
Scenario: A byte-fallback token that completes a multi-byte
UTF-8 sequence when decoded with context.
"""
def test_correction_with_previous_token_in_list(
self, processor_with_empty_logprobs
):
"""Test correction using previous token in the same list.
# Context is [101] (a preceding sampled token)
# Token 102 individually decodes to "�"
# decode([101, 102]) returns "valid" (complete sequence)
def mock_decode(ids):
if ids == [101, 102]:
return "hello valid"
if ids == [101]:
return "hello "
return "�"
Scenario: Token at idx=1 ends with "�", but when decoded with
the previous token (idx=0), it forms a valid UTF-8 sequence.
Example: token[0]="�", token[1]="�" -> together form "polarized"
"""
processor = processor_with_empty_logprobs
tokens = [100, 101, 102] # token IDs
# Mock tokenizer behavior:
# - decode([102]) returns "�" (ends with replacement char)
# - decode([101, 102]) returns "valid" (no replacement char)
processor.tokenizer.decode.side_effect = lambda ids: (
"valid" if ids == [101, 102] else "�"
)
processor.tokenizer.decode.side_effect = mock_decode
result = processor._correct_decoded_token(2, tokens)
result = processor._correct_decoded_token(102, [101])
assert result == "valid"
processor.tokenizer.decode.assert_called_with([101, 102])
def test_correction_with_previous_logprob_token(
self, processor_with_previous_logprobs
):
"""Test correction using previous logprob token.
def test_correction_with_context_from_logprobs(self, processor):
"""Test correction using context from previous logprob entries.
Scenario: Cannot correct with previous token in list (idx=0),
but can correct with previous logprob token.
Scenario: Token decoded with context from previously sampled
tokens completes a UTF-8 sequence.
"""
processor = processor_with_previous_logprobs
tokens = [100] # single token
# Mock tokenizer behavior:
# - decode([100]) returns "�" (ends with replacement char)
# - decode([123, 100]) returns " "polarized" (no replacement char)
# Token 123 is from previous logprobs
# Token 123 was previously sampled (in context)
def mock_decode(ids):
if ids == [123, 100]:
return ' "polarized"'
return 'hello "polarized"'
if ids == [123]:
return "hello "
return "�"
processor.tokenizer.decode.side_effect = mock_decode
result = processor._correct_decoded_token(0, tokens)
assert result == ' "polarized"'
result = processor._correct_decoded_token(100, [123])
assert result == '"polarized"'
def test_correction_at_idx_zero_no_previous_logprobs(
self, processor_with_empty_logprobs
):
"""Test correction at idx=0 with no previous logprobs.
def test_correction_no_context(self, processor):
"""Test correction with no context available.
Scenario: First token in list, no previous logprobs available.
Should return empty string as fallback.
"""
processor = processor_with_empty_logprobs
tokens = [100]
# Mock tokenizer always returns "�"
processor.tokenizer.decode.return_value = "�"
result = processor._correct_decoded_token(0, tokens)
result = processor._correct_decoded_token(100, [])
assert result == ""
def test_correction_at_idx_zero_with_previous_logprobs(
self, processor_with_previous_logprobs
):
"""Test correction at idx=0 with previous logprobs available.
def test_correction_with_context_succeeds(self, processor):
"""Test correction with context from previously sampled tokens."""
Scenario: First token in list, but previous logprobs exist.
Should try correction with previous logprob token.
"""
processor = processor_with_previous_logprobs
tokens = [200]
# Mock tokenizer behavior
def mock_decode(ids):
if ids == [123, 200]:
return "corrected"
return "hello corrected"
if ids == [123]:
return "hello "
return "�"
processor.tokenizer.decode.side_effect = mock_decode
result = processor._correct_decoded_token(0, tokens)
result = processor._correct_decoded_token(200, [123])
assert result == "corrected"
def test_no_correction_needed_returns_fallback(
self, processor_with_previous_logprobs
):
"""Test fallback to empty string when no correction works.
Scenario: All correction attempts still end with "�".
Should return empty string as final fallback.
"""
processor = processor_with_previous_logprobs
tokens = [100, 101, 102]
# Mock tokenizer always returns text ending with "�"
def test_fallback_when_all_attempts_fail(self, processor):
"""Test fallback to empty string when no correction works."""
processor.tokenizer.decode.return_value = "still�"
result = processor._correct_decoded_token(2, tokens)
result = processor._correct_decoded_token(102, [100, 101])
assert result == ""
def test_middle_token_correction(self, processor_with_previous_logprobs):
"""Test correction for a token in the middle of the list.
def test_increasing_context_window(self, processor):
"""Test that increasing context window finds the correction.
Scenario: Token at idx=5 in a longer list needs correction.
Scenario: 3-byte UTF-8 char. With 1 context token, still
incomplete. With 2 context tokens, completes the sequence.
"""
processor = processor_with_previous_logprobs
tokens = [10, 20, 30, 40, 50, 60, 70, 80]
# Mock tokenizer behavior for middle token
def mock_decode(ids):
if ids == [50, 60]:
return "olar"
# 1 context token: still incomplete
if ids == [81, 82]:
return "�"
# 2 context tokens: complete
if ids == [80, 81, 82]:
return "\u201c"
# Context-only decodes
if ids == [81]:
return "�"
if ids == [80, 81]:
return "�"
return "�"
processor.tokenizer.decode.side_effect = mock_decode
result = processor._correct_decoded_token(5, tokens)
assert result == "olar"
# Context has 2 preceding tokens [80, 81]
result = processor._correct_decoded_token(82, [80, 81])
assert result == "\u201c"
def test_multiple_consecutive_replacement_chars(
self, processor_with_previous_logprobs
):
def test_multiple_consecutive_replacement_chars(self, processor):
"""Test handling of multiple consecutive replacement characters.
Scenario: Sequence like ["�", "�", "p"] where first two should
become empty strings.
Scenario: Multi-byte sequence where intermediate bytes return
empty string and the final byte returns the complete character.
"""
processor = processor_with_previous_logprobs
# Test first replacement char
tokens = [100, 101, 102]
processor.tokenizer.decode.return_value = "still�"
result1 = processor._correct_decoded_token(0, tokens)
# First byte with no useful context: returns ""
result1 = processor._correct_decoded_token(100, [50])
assert result1 == ""
# Test second replacement char
result2 = processor._correct_decoded_token(1, tokens)
# Second byte with same context: still returns ""
result2 = processor._correct_decoded_token(101, [50])
assert result2 == ""
def test_correction_with_multibyte_utf8(self, processor_with_previous_logprobs):
def test_correction_with_multibyte_utf8(self, processor):
"""Test correction involving multi-byte UTF-8 characters.
Scenario: Byte-fallback tokenization splits multi-byte UTF-8
characters (e.g., curly quotes, Chinese characters, emojis).
Example from user: "�", "�" -> "", "\""
Scenario: Byte-fallback tokenization splits curly quotes.
The last byte token should produce the complete character.
"""
processor = processor_with_previous_logprobs
tokens = [200, 201]
# Mock tokenizer behavior for multi-byte UTF-8 correction
def mock_decode(ids):
# When decoding first token (idx=0) with previous logprob token
# Context [123] + first byte: completes to left curly quote
if ids == [123, 200]:
return ' "' # Space + left curly quote
# When decoding second token (idx=1) with previous token in list
elif ids == [200, 201]:
return '"' # Right curly quote
# When decoding second token (idx=1) with previous logprob + prev token
elif ids == [123, 200, 201]:
return ' ""' # Full sequence
return "�"
return "hello \u201c"
if ids == [123]:
return "hello "
# Context [123] + second byte: completes to right curly quote
if ids == [123, 201]:
return "hello \u201d"
return "\ufffd"
processor.tokenizer.decode.side_effect = mock_decode
# First token correction (idx=0)
# Will call decode([123, 200]) since idx=0 uses previous logprob token
result1 = processor._correct_decoded_token(0, tokens)
assert result1 == ' "'
# Each top-k token is corrected independently with same context
result1 = processor._correct_decoded_token(200, [123])
assert result1 == "\u201c"
# Second token correction (idx=1)
# Will call decode([200, 201]) since idx>0 uses previous token in list
result2 = processor._correct_decoded_token(1, tokens)
assert result2 == '"'
result2 = processor._correct_decoded_token(201, [123])
assert result2 == "\u201d"
def test_topk_tokens_corrected_independently(self, processor):
"""Test that top-k alternatives at the same position are each
corrected independently using only sequential context, not
each other.
This is the core fix for issue #27300: when logprobs > 0,
alternative tokens must not be combined with each other.
"""
# Context: previously sampled token 50
context = [50]
def mock_decode(ids):
# Token 100 (sampled) with context
if ids == [50, 100]:
return "prefix \u201c"
# Token 200 (top-k alternative) with context
if ids == [50, 200]:
return "prefix \u2014"
# Context alone
if ids == [50]:
return "prefix "
return "\ufffd"
processor.tokenizer.decode.side_effect = mock_decode
# Both tokens at the same position use the SAME context [50]
result_sampled = processor._correct_decoded_token(100, context)
assert result_sampled == "\u201c"
result_alt = processor._correct_decoded_token(200, context)
assert result_alt == "\u2014"
def test_real_world_opt125m_scenario(self, mock_tokenizer):
"""Test the real-world scenario from user's example.
"""Test the real-world scenario from the bug report.
User's example with facebook/opt-125m:
Before: [" the", " term", " �", "�", "p", "olar", "ized", "�", "�", ...]
After: [" the", " term", "", " "", "p", "olar", "ized", "", "\"", ...]
Simulates the OPT-125m sequence where curly quotes are split
into byte-fallback tokens. Each token is corrected using only
the preceding sampled tokens as context.
"""
from vllm.v1.engine.logprobs import LogprobsProcessor
# Simulate the sequence of tokens
processor = LogprobsProcessor(
tokenizer=mock_tokenizer,
logprobs=[],
......@@ -776,47 +762,106 @@ class TestCorrectDecodedToken:
num_prompt_logprobs=None,
)
# Token IDs representing the problematic sequence
tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9] # placeholder IDs
# Mock decode behavior simulating the real scenario
# Simulating: byte tokens 3, 4 form left curly quote "\u201c"
# byte tokens 8, 9 form right curly quote "\u201d"
def mock_decode(ids):
# Simulate cases where individual tokens decode to "�"
# but combinations decode correctly
if len(ids) == 1:
if ids[0] in (3, 4, 8, 9):
return "�"
elif len(ids) == 2:
# Context decodes
if ids == [2]:
return " term"
if ids == [1, 2]:
return " the term"
if ids == [3]:
return "\ufffd"
if ids == [2, 3]:
return " term�" # Still ends with �, need more context
elif ids == [3, 4]:
return ' "' # Corrected to space + left curly quote
elif ids == [7, 8]:
return "ized�" # Still ends with �
elif ids == [8, 9]:
return '"' # Corrected to right curly quote
elif len(ids) == 3:
return " term\ufffd"
if ids == [1, 2, 3]:
return " the term�" # Still ends with issue
elif ids == [2, 3, 4]:
return ' term "' # With all context
return " the term\ufffd"
# Token 4 with context [2, 3] -> completes left curly quote
if ids == [3, 4]:
return "\u201c"
if ids == [2, 3, 4]:
return " term\u201c"
# Context for right curly quote
if ids == [7]:
return "ized"
if ids == [7, 8]:
return "ized\ufffd"
if ids == [8, 9]:
return "\u201d"
if ids == [7, 8, 9]:
return "ized\u201d"
return "normal_text"
mock_tokenizer.decode.side_effect = mock_decode
# Test token at index 2 (should fail to correct, return "")
# Token 3 individually is "�"
# decode([2, 3]) = " term�" (still ends with �)
# No previous logprobs, so fallback to ""
result = processor._correct_decoded_token(2, tokens)
# First byte (token 3) of left curly quote with no context
result = processor._correct_decoded_token(3, [])
assert result == ""
# First byte (token 3) with context [2] -> still incomplete
result = processor._correct_decoded_token(3, [2])
assert result == ""
# Second byte (token 4) of left curly quote with context [2, 3]
# Token 3 is byte-fallback, so clean context is [2] only.
# decode([2, 3, 4]) = " term\u201c", decode([2]) = " term"
# result = "\u201c"
result = processor._correct_decoded_token(4, [2, 3])
assert result == "\u201c"
# Second byte (token 9) of right curly quote with context [7, 8]
result = processor._correct_decoded_token(9, [7, 8])
assert result == "\u201d"
def test_byte_fallback_context_preserves_space(self, mock_tokenizer):
"""Test that text from byte-fallback context tokens is preserved.
In OPT-125m, token 44 = space + 2 bytes of curly quote.
When token 44 returns "" (incomplete), the space it carried
must be attributed to the completing token (48).
"""
from vllm.v1.engine.logprobs import LogprobsProcessor
processor = LogprobsProcessor(
tokenizer=mock_tokenizer,
logprobs=[],
prompt_logprobs=None,
cumulative_logprob=0.0,
num_logprobs=1,
num_prompt_logprobs=None,
)
def mock_decode(ids):
# Token 44 = space + 2 bytes (like OPT-125m's \u0120\u00e2\u0080)
if ids == [44]:
return " \ufffd"
if ids == [48]:
return "\ufffd"
# Together they form: space + left curly quote
if ids == [44, 48]:
return " \u201c"
# With preceding clean context
if ids == [1385]:
return " term"
if ids == [1385, 44]:
return " term \ufffd"
if ids == [1385, 44, 48]:
return " term \u201c"
return "\ufffd"
mock_tokenizer.decode.side_effect = mock_decode
# Token 44 with context [1385] -> still ends with replacement
result = processor._correct_decoded_token(44, [1385])
assert result == ""
# Test token at index 3 (should correct to " "")
# Token 4 individually is "�"
# decode([3, 4]) = " "" (corrected!)
processor.logprobs = [{2: None}] # Add previous logprob
result = processor._correct_decoded_token(3, tokens)
assert result == ' "'
# Token 48 with context [1385, 44]:
# Token 44 is byte-fallback, so clean context is [1385].
# decode([1385, 44, 48]) = " term \u201c"
# decode([1385]) = " term"
# result = " \u201c" (space preserved from token 44!)
result = processor._correct_decoded_token(48, [1385, 44])
assert result == " \u201c"
def test_verify_tokens_integration():
......
vllm/v1/engine/logprobs.py
View file @ 8904fc4d
......@@ -7,6 +7,7 @@ from dataclasses import dataclass
from vllm.logger import init_logger
from vllm.logprobs import (
FlatLogprobs,
PromptLogprobs,
SampleLogprobs,
append_logprobs_for_next_position,
......@@ -96,8 +97,11 @@ class LogprobsProcessor:
decoded_tokens_list = convert_ids_list_to_tokens(
self.tokenizer, token_ids
)
context_token_ids = self._get_sampled_context_ids(self.logprobs)
decoded_tokens = self._verify_tokens(
decoded_tokens_list=decoded_tokens_list, tokens=token_ids
decoded_tokens_list=decoded_tokens_list,
tokens=token_ids,
context_token_ids=context_token_ids,
)
# Sampler puts the sampled logprob in first.
......@@ -162,9 +166,14 @@ class LogprobsProcessor:
else:
# Extract decoded tokens for this position
decoded_tokens_slice = all_decoded_tokens[offset:offset_end]
# Context: preceding prompt tokens accumulated in
# self.prompt_logprobs from previous loop iterations.
context_token_ids = self._get_sampled_context_ids(self.prompt_logprobs)
# Apply UTF-8 correction within this position's token boundaries
decoded_tokens_for_pos = self._verify_tokens(
decoded_tokens_list=decoded_tokens_slice, tokens=token_ids_list[pos]
decoded_tokens_list=decoded_tokens_slice,
tokens=token_ids_list[pos],
context_token_ids=context_token_ids,
)
# Update with the Logprob container for this pos.
......@@ -196,41 +205,139 @@ class LogprobsProcessor:
self.prompt_logprobs = []
return plp
def _correct_decoded_token(self, idx: int, tokens: list[int]) -> str:
assert self.tokenizer is not None, "self.tokenizer should not be None"
# try with prev token id in same list
if idx > 0:
possible_decoded_token = self.tokenizer.decode(tokens[idx - 1 : idx + 1])
if not possible_decoded_token.endswith("�"):
return possible_decoded_token
# try with previous logprob token id
if self.logprobs:
latest_token_id = next(iter(self.logprobs[-1]))
decode_ids = [latest_token_id]
if idx > 0:
decode_ids.extend(tokens[idx - 1 : idx + 1])
@staticmethod
def _get_sampled_context_ids(
logprobs_source: SampleLogprobs | PromptLogprobs | None,
max_context: int = 4,
) -> list[int]:
"""Extract recent sampled token IDs from a logprobs source.
The sampled (or prompt) token at each position is the first
entry, since it is always inserted first by
append_logprobs_for_next_position.
Args:
logprobs_source: The logprobs container to extract from.
max_context: Maximum number of preceding tokens to return.
4 is sufficient for any UTF-8 multi-byte sequence.
Returns:
List of sampled token IDs, oldest first, most recent last.
"""
if not logprobs_source:
return []
n = len(logprobs_source)
start = max(0, n - max_context)
# Efficient path for FlatLogprobs: access token_ids directly.
if isinstance(logprobs_source, FlatLogprobs):
return [
logprobs_source.token_ids[logprobs_source.start_indices[i]]
for i in range(start, n)
if logprobs_source.start_indices[i] < logprobs_source.end_indices[i]
]
# list[dict] path
result: list[int] = []
for i in range(start, n):
entry = logprobs_source[i]
if entry is not None:
result.append(next(iter(entry)))
return result
def _correct_decoded_token(
self, token_id: int, context_token_ids: list[int]
) -> str:
"""Correct a decoded token that contains the replacement character.
When byte-fallback tokenization splits multi-byte UTF-8
characters across tokens, individual token decoding produces
the replacement character U+FFFD. This method uses preceding
sampled tokens as context to reconstruct the correct text.
Args:
token_id: The single token ID to correct.
context_token_ids: Preceding sampled token IDs in sequential
order (oldest first). These are the actual tokens in
the generated sequence, NOT top-k alternatives.
Returns:
The corrected decoded string, or empty string if the byte
sequence is genuinely incomplete at this point.
"""
assert self.tokenizer is not None
max_ctx = min(len(context_token_ids), 4)
for num_ctx in range(1, max_ctx + 1):
context = context_token_ids[-num_ctx:]
full_decoded = self.tokenizer.decode(context + [token_id])
if full_decoded.endswith("�"):
continue
# Find the boundary between "clean" context tokens and
# byte-fallback tokens that are part of the same incomplete
# sequence. Byte-fallback context tokens returned "" when
# they were processed, so their text must be attributed to
# this completing token.
clean_end = len(context)
for j in range(len(context) - 1, -1, -1):
if self.tokenizer.decode([context[j]]).endswith("�"):
clean_end = j
else:
break
# Decode only the clean (non-byte-fallback) prefix.
if clean_end > 0:
clean_prefix = self.tokenizer.decode(context[:clean_end])
else:
decode_ids.extend(tokens[idx : idx + 1])
clean_prefix = ""
possible_decoded_token = self.tokenizer.decode(decode_ids)
if not possible_decoded_token.endswith("�"):
return possible_decoded_token
if full_decoded.startswith(clean_prefix):
return full_decoded[len(clean_prefix) :]
# Tokenizer normalization may cause prefix mismatch.
# Find the longest common prefix between them.
common_len = 0
for a, b in zip(clean_prefix, full_decoded):
if a != b:
break
common_len += 1
return full_decoded[common_len:]
# by default return empty string
return ""
def _verify_tokens(
self, decoded_tokens_list: list[str], tokens: list[int]
self,
decoded_tokens_list: list[str],
tokens: list[int],
context_token_ids: list[int] | None = None,
) -> list[str]:
"""Verify and correct decoded tokens with replacement characters.
Args:
decoded_tokens_list: Decoded token strings to verify.
tokens: Token IDs corresponding to decoded_tokens_list.
These are alternatives at the SAME position (e.g.
[sampled, top1, top2]), NOT sequential tokens.
context_token_ids: Preceding sampled token IDs providing
sequential context. If None, extracted from
self.logprobs.
"""
if context_token_ids is None:
context_token_ids = self._get_sampled_context_ids(self.logprobs)
corrected_decoded_token_map = dict()
for idx, text in enumerate(decoded_tokens_list):
if text.endswith("�"):
# utf-8 char at the end means it's a potential unfinished byte sequence
# from byte fallback tokenization.
# Replacement char at the end means a potential
# unfinished byte sequence from byte-fallback
# tokenization. Correct each token independently
# using only the sequential context.
corrected_decoded_token_map[idx] = self._correct_decoded_token(
idx, tokens
tokens[idx], context_token_ids
)
for idx, text in corrected_decoded_token_map.items():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment