Unverified Commit af376ca1 authored by Jialin Ouyang's avatar Jialin Ouyang Committed by GitHub
Browse files

[Core] Minimize number of dict lookup in _maybe_evict_cached_block (#21281)


Signed-off-by: default avatarJialin Ouyang <Jialin.Ouyang@gmail.com>
parent e7b20426
...@@ -243,22 +243,27 @@ class BlockPool: ...@@ -243,22 +243,27 @@ class BlockPool:
True if the block is evicted, False otherwise. True if the block is evicted, False otherwise.
""" """
block_hash = block.block_hash block_hash = block.block_hash
if block_hash and block_hash in self.cached_block_hash_to_block: if block_hash is None:
block.reset_hash() # The block doesn't have hash, eviction is not needed
del self.cached_block_hash_to_block[block_hash][block.block_id] return False
blocks_by_id = self.cached_block_hash_to_block.get(block_hash)
if len(self.cached_block_hash_to_block[block_hash]) == 0: if blocks_by_id is None:
del self.cached_block_hash_to_block[block_hash] # block_hash not found in cached_block_hash_to_block,
# eviction is not needed
if self.enable_kv_cache_events: return False
# FIXME (Chen): Not sure whether we should return `hash_value` block.reset_hash()
# or `(hash_value, group_id)` here. But it's fine now because blocks_by_id.pop(block.block_id, None)
# we disable hybrid kv cache manager when kv cache event is if blocks_by_id:
# enabled, so there is only one group. del self.cached_block_hash_to_block[block_hash]
self.kv_event_queue.append(
BlockRemoved(block_hashes=[block_hash.get_hash_value()])) if self.enable_kv_cache_events:
return True # FIXME (Chen): Not sure whether we should return `hash_value`
return False # or `(hash_value, group_id)` here. But it's fine now because
# we disable hybrid kv cache manager when kv cache event is
# enabled, so there is only one group.
self.kv_event_queue.append(
BlockRemoved(block_hashes=[block_hash.get_hash_value()]))
return True
def touch(self, blocks: tuple[list[KVCacheBlock], ...]) -> None: def touch(self, blocks: tuple[list[KVCacheBlock], ...]) -> None:
"""Touch a block increases its reference count by 1, and may remove """Touch a block increases its reference count by 1, and may remove
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment