Unverified Commit 6e073516 authored by Alec's avatar Alec Committed by GitHub
Browse files

fix: indenting mistake missed condition. (#2825)


Signed-off-by: default avataralec-flowers <aflowers@nvidia.com>
parent 383e3b3a
...@@ -116,6 +116,8 @@ class DecodeWorkerHandler(BaseWorkerHandler): ...@@ -116,6 +116,8 @@ class DecodeWorkerHandler(BaseWorkerHandler):
if self.prefill_worker_client is not None: if self.prefill_worker_client is not None:
self.can_prefill = len(self.prefill_worker_client.instance_ids()) self.can_prefill = len(self.prefill_worker_client.instance_ids())
logger.debug(f"Current Prefill Workers: {self.can_prefill}") logger.debug(f"Current Prefill Workers: {self.can_prefill}")
else:
self.can_prefill = 0
except asyncio.CancelledError: except asyncio.CancelledError:
logger.warning("Prefill check loop cancelled.") logger.warning("Prefill check loop cancelled.")
raise raise
...@@ -147,6 +149,7 @@ class DecodeWorkerHandler(BaseWorkerHandler): ...@@ -147,6 +149,7 @@ class DecodeWorkerHandler(BaseWorkerHandler):
if value is not None and hasattr(sampling_params, key): if value is not None and hasattr(sampling_params, key):
setattr(sampling_params, key, value) setattr(sampling_params, key, value)
# TODO Change to prefill queue
if self.can_prefill: if self.can_prefill:
# Create a copy for prefill with specific modifications # Create a copy for prefill with specific modifications
prefill_sampling_params = deepcopy(sampling_params) prefill_sampling_params = deepcopy(sampling_params)
...@@ -165,8 +168,6 @@ class DecodeWorkerHandler(BaseWorkerHandler): ...@@ -165,8 +168,6 @@ class DecodeWorkerHandler(BaseWorkerHandler):
"request_id": request_id, "request_id": request_id,
} }
# TODO Change to prefill queue
if self.prefill_worker_client is not None:
try: try:
prefill_response = await anext( prefill_response = await anext(
await self.prefill_worker_client.round_robin( await self.prefill_worker_client.round_robin(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment