"docs/API/vscode:/vscode.git/clone" did not exist on "844f8819a1af7dce6eff2d0b7911b1ed1c018ab3"
Unverified Commit a7ea35aa authored by Tyler Michael Smith's avatar Tyler Michael Smith Committed by GitHub
Browse files

[Bugfix] Remove num_tokens_across_dp (#14302)


Signed-off-by: default avatarTyler Michael Smith <tyler@neuralmagic.com>
parent 1e3e76b6
......@@ -27,7 +27,6 @@ batchsize_forward_time: defaultdict = defaultdict(list)
@dataclass
class DPMetadata:
num_tokens_across_dp: list[int]
cu_tokens_across_dp_cpu: torch.Tensor
......@@ -89,7 +88,7 @@ def set_forward_context(attn_metadata: Any,
from vllm.distributed.parallel_state import get_dp_group
dist.all_reduce(num_tokens_tensor, group=get_dp_group().cpu_group)
cu_tokens_across_dp_cpu = torch.cumsum(num_tokens_tensor, dim=0)
dp_metadata = DPMetadata(num_tokens_across_dp, cu_tokens_across_dp_cpu)
dp_metadata = DPMetadata(cu_tokens_across_dp_cpu)
global _forward_context
prev_context = _forward_context
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment