"lib/bindings/vscode:/vscode.git/clone" did not exist on "9c08a2aa032c721369b7a5a0eaac0bb5f99e5cb9"
Unverified Commit 173201f5 authored by Alec's avatar Alec Committed by GitHub
Browse files

fix: vLLM xfer bug (#4745)


Signed-off-by: default avataralec-flowers <aflowers@nvidia.com>
parent 8601ccdb
......@@ -4,7 +4,10 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional, Type
from kvbm.vllm_integration.connector.dynamo_connector import DynamoConnector
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorHandshakeMetadata,
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
MultiConnector,
MultiKVConnectorMetadata,
......@@ -84,6 +87,18 @@ class PdConnector(MultiConnector):
# Worker-side methods
# ==============================
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
) -> None:
"""
Propagate handshake metadata to child connectors.
This is required for NIXL connector to start its handshake listener
which decode workers connect to for KV transfer coordination.
"""
for c in self._connectors:
c.set_xfer_handshake_metadata(metadata)
def bind_connector_metadata(self, connector_metadata: PdConnectorMetadata) -> None:
assert isinstance(connector_metadata, PdConnectorMetadata)
if connector_metadata.extra_async_saves:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment