Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OpenDAS
vllm_cscc
Commits
c6b0a7d3
Unverified
Commit
c6b0a7d3
authored
Dec 18, 2024
by
Chen Zhang
Committed by
GitHub
Dec 19, 2024
Browse files
[V1] Simplify prefix caching logic by removing `num_evictable_computed_blocks` (#11310)
parent
a30482f0
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
2 additions
and
11 deletions
+2
-11
vllm/v1/core/kv_cache_manager.py
vllm/v1/core/kv_cache_manager.py
+2
-11
No files found.
vllm/v1/core/kv_cache_manager.py
View file @
c6b0a7d3
...
...
@@ -201,23 +201,15 @@ class KVCacheManager:
f
"num_tokens must be greater than 0, got
{
num_tokens
}
"
)
# Touch the computed blocks to make sure they won't be evicted.
num_evictable_computed_blocks
=
0
if
self
.
enable_caching
:
self
.
_touch
(
computed_blocks
)
# If a computed block of a request is an eviction candidate (in the
# free queue and ref_cnt == 0), it cannot be counted as a free block
# when allocating this request.
num_evictable_computed_blocks
=
len
(
[
blk
for
blk
in
computed_blocks
if
blk
.
ref_cnt
==
0
])
else
:
assert
not
computed_blocks
,
(
"Computed blocks should be empty when "
"prefix caching is disabled"
)
num_required_blocks
=
cdiv
(
num_tokens
,
self
.
block_size
)
if
(
num_required_blocks
>
self
.
free_block_queue
.
num_free_blocks
-
num_evictable_computed_blocks
):
if
(
num_required_blocks
>
self
.
free_block_queue
.
num_free_blocks
):
# Cannot allocate new blocks.
return
None
...
...
@@ -225,8 +217,7 @@ class KVCacheManager:
# preallocated blocks.
num_new_blocks
=
min
(
num_required_blocks
+
self
.
num_preallocate_blocks
,
self
.
free_block_queue
.
num_free_blocks
-
num_evictable_computed_blocks
,
self
.
free_block_queue
.
num_free_blocks
,
# Should not exceed the maximum number of blocks per request.
# This is especially because the block table has the shape
# [..., max_num_blocks_per_req].
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment