Skip to content

GitLab

Unverified Commit cea96a01 authored by Rui Qiao's avatar Rui Qiao Committed by GitHub
Browse files

[Bugfix] Fix sync_and_slice_intermediate_tensors (#21537) 


Signed-off-by: default avatarRui Qiao <ruisearch42@gmail.com>
parent 2eddd437
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
vllm/v1/worker/gpu_model_runner.py
View file @ cea96a01
...@@ -1270,7 +1270,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): ...@@ -1270,7 +1270,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
if sync_self: if sync_self:
assert intermediate_tensors is not None assert intermediate_tensors is not None
for k, v in intermediate_tensors.items(): for k, v in intermediate_tensors.items():
is_scattered = "residual" and is_residual_scattered is_scattered = k == "residual" and is_residual_scattered
copy_len = num_tokens // tp if is_scattered else \ copy_len = num_tokens // tp if is_scattered else \
num_tokens num_tokens
self.intermediate_tensors[k][:copy_len].copy_( self.intermediate_tensors[k][:copy_len].copy_(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment