Commit a857453f authored by zhuwenwen's avatar zhuwenwen
Browse files

Merge branch 'v0.9.2-dev_pp_mtp' into 'v0.9.2-dev'

[fix] pp+mtp bs 1 correctness

See merge request dcutoolkit/deeplearing/vllm!214
parents c1ece8c6 22d6f9af
......@@ -528,10 +528,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not is_last_rank:
# Add new_token_ids to token_ids_cpu.
start_token_index = num_computed_tokens
end_token_index = num_computed_tokens + len(new_token_ids)
end_token_index = num_computed_tokens + 1
self.input_batch.token_ids_cpu[
req_index,
start_token_index:end_token_index] = new_token_ids
start_token_index:end_token_index] = new_token_ids[-1]
self.input_batch.num_tokens_no_spec[
req_index] = end_token_index
self.input_batch.num_tokens[req_index] = end_token_index
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment