"lib/vscode:/vscode.git/clone" did not exist on "f78189d7dae46ee32fc4b9065ad2b8799ae185b5"
Unverified Commit d7c27e68 authored by jthomson04's avatar jthomson04 Committed by GitHub
Browse files

fix: Fix TRTLLM disagg with kv router (#4950)


Signed-off-by: default avatarjthomson04 <jwillthomson19@gmail.com>
Co-authored-by: default avatarRyan McCormick <rmccormick@nvidia.com>
parent 036eb4fe
...@@ -283,6 +283,10 @@ class HandlerBase: ...@@ -283,6 +283,10 @@ class HandlerBase:
if "prefill_result" in request: if "prefill_result" in request:
if self.disaggregation_mode == DisaggregationMode.PREFILL: if self.disaggregation_mode == DisaggregationMode.PREFILL:
raise ValueError("Cannot provide disaggregated_params in prefill mode") raise ValueError("Cannot provide disaggregated_params in prefill mode")
request["prefill_result"].get("disaggregated_params", {}).pop(
"worker_id", None
)
disaggregated_params = DisaggregatedParamsCodec.decode( disaggregated_params = DisaggregatedParamsCodec.decode(
DisaggregatedParams( DisaggregatedParams(
**request["prefill_result"].get("disaggregated_params") **request["prefill_result"].get("disaggregated_params")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment