Skip to content

GitLab

Unverified Commit 660470e5 authored by xiaobochen123's avatar xiaobochen123 Committed by GitHub
Browse files

[Core] Optimize evictor-v2 performance (#7193)

parent 8d59dbb0
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 2 deletions
vllm/core/evictor_v2.py
View file @ 660470e5
...@@ -91,8 +91,9 @@ class LRUEvictor(Evictor): ...@@ -91,8 +91,9 @@ class LRUEvictor(Evictor):
# at the start of OrderedDict. Loop through all these blocks to # at the start of OrderedDict. Loop through all these blocks to
# find the one with maximum number of hashed tokens. # find the one with maximum number of hashed tokens.
for _id, block in self.free_table.items(): for _id, block in self.free_table.items():
if evicted_block.last_accessed > block.last_accessed or ( if evicted_block.last_accessed < block.last_accessed:
evicted_block.last_accessed == block.last_accessed and break
if (evicted_block.last_accessed == block.last_accessed and
evicted_block.num_hashed_tokens < block.num_hashed_tokens): evicted_block.num_hashed_tokens < block.num_hashed_tokens):
evicted_block = block evicted_block = block
evicted_block_id = _id evicted_block_id = _id
...@@ -109,6 +110,7 @@ class LRUEvictor(Evictor): ...@@ -109,6 +110,7 @@ class LRUEvictor(Evictor):
def update(self, block_id: int, last_accessed: float): def update(self, block_id: int, last_accessed: float):
self.free_table[block_id].last_accessed = last_accessed self.free_table[block_id].last_accessed = last_accessed
self.free_table.move_to_end(block_id)
def remove(self, block_id: int): def remove(self, block_id: int):
if block_id not in self.free_table: if block_id not in self.free_table:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment