Unverified Commit 3029d301 authored by cctry's avatar cctry Committed by GitHub
Browse files

Fix crash after flush cache (#12107)


Co-authored-by: default avatarfzyzcjy <5236035+fzyzcjy@users.noreply.github.com>
parent f389f017
......@@ -2305,13 +2305,30 @@ class Scheduler(
if_success = False
return ClearHiCacheReqOutput(success=if_success)
def flush_cache(self):
"""Flush the memory pool and cache."""
if (
def _is_no_request(self):
no_request = (
len(self.waiting_queue) == 0
and self.running_batch.is_empty()
and (self.last_batch is None or self.last_batch.is_empty())
and (self.cur_batch is None or self.cur_batch.is_empty())
and (not self.enable_overlap or len(self.result_queue) == 0)
and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
):
)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
no_request &= (
len(self.disagg_prefill_bootstrap_queue.queue) == 0
and len(self.disagg_prefill_inflight_queue) == 0
)
if self.disaggregation_mode == DisaggregationMode.DECODE:
no_request &= (
len(self.disagg_decode_prealloc_queue.queue) == 0
and len(self.disagg_decode_transfer_queue.queue) == 0
)
return no_request
def flush_cache(self):
"""Flush the memory pool and cache."""
if self._is_no_request():
self.cur_batch = None
self.last_batch = None
self.tree_cache.reset()
......
......@@ -533,6 +533,10 @@ class RadixCache(BasePrefixCache):
self.protected_size_ -= len(node.key)
delta += len(node.key)
node.lock_ref -= 1
if node.parent is None:
assert (
node is self.root_node
), f"This request holds the node from another tree"
node = node.parent
return delta
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment