We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 1e3e76b commit a7ea35aCopy full SHA for a7ea35a
vllm/forward_context.py
@@ -27,7 +27,6 @@
27
28
@dataclass
29
class DPMetadata:
30
- num_tokens_across_dp: list[int]
31
cu_tokens_across_dp_cpu: torch.Tensor
32
33
@@ -89,7 +88,7 @@ def set_forward_context(attn_metadata: Any,
89
88
from vllm.distributed.parallel_state import get_dp_group
90
dist.all_reduce(num_tokens_tensor, group=get_dp_group().cpu_group)
91
cu_tokens_across_dp_cpu = torch.cumsum(num_tokens_tensor, dim=0)
92
- dp_metadata = DPMetadata(num_tokens_across_dp, cu_tokens_across_dp_cpu)
+ dp_metadata = DPMetadata(cu_tokens_across_dp_cpu)
93
94
global _forward_context
95
prev_context = _forward_context
0 commit comments