Unverified Commit bd835b03 authored by Ziqi Fan's avatar Ziqi Fan Committed by GitHub
Browse files

fix: change KVBM Connector build_connector_meta due to vLLM ToT change (#2887)


Signed-off-by: default avatarZiqi Fan <ziqif@nvidia.com>
parent c8ecc402
......@@ -142,6 +142,7 @@ class KvConnectorLeader:
scheduler_output.scheduled_cached_reqs.new_block_ids,
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
):
if new_block_ids is not None:
output.add_cached_request(
request_id=req_id,
resumed_from_preemption=resumed_from_preemption,
......@@ -149,6 +150,14 @@ class KvConnectorLeader:
new_block_ids=new_block_ids[0],
num_computed_tokens=num_computed_tokens,
)
else:
output.add_cached_request(
request_id=req_id,
resumed_from_preemption=resumed_from_preemption,
new_token_ids=new_token_ids,
new_block_ids=[],
num_computed_tokens=num_computed_tokens,
)
output.add_num_scheduled_tokens(scheduler_output.num_scheduled_tokens)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment