Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
zhaoyu6
sglang
Commits
5b647543
"vscode:/vscode.git/clone" did not exist on "5e2f229561760f6b85302ccb0ab42db0dbf1731a"
Unverified
Commit
5b647543
authored
May 19, 2024
by
Liangsheng Yin
Committed by
GitHub
May 19, 2024
Browse files
Fix the broken `--disable-radix-cache` (#451)
parent
8210ec60
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
7 additions
and
1 deletion
+7
-1
python/sglang/srt/managers/router/radix_cache.py
python/sglang/srt/managers/router/radix_cache.py
+7
-1
No files found.
python/sglang/srt/managers/router/radix_cache.py
View file @
5b647543
...
@@ -58,7 +58,7 @@ class RadixCache:
...
@@ -58,7 +58,7 @@ class RadixCache:
def
insert
(
self
,
key
,
value
=
None
):
def
insert
(
self
,
key
,
value
=
None
):
if
self
.
disable
:
if
self
.
disable
:
return
len
(
key
)
return
0
if
value
is
None
:
if
value
is
None
:
value
=
[
x
for
x
in
key
]
value
=
[
x
for
x
in
key
]
...
@@ -76,6 +76,12 @@ class RadixCache:
...
@@ -76,6 +76,12 @@ class RadixCache:
indices
=
self
.
req_to_token_pool
.
req_to_token
[
req_pool_idx
,
:
len
(
token_ids
)]
indices
=
self
.
req_to_token_pool
.
req_to_token
[
req_pool_idx
,
:
len
(
token_ids
)]
new_prefix_len
=
self
.
insert
(
token_ids
,
indices
.
clone
())
new_prefix_len
=
self
.
insert
(
token_ids
,
indices
.
clone
())
if
self
.
disable
:
if
del_in_memory_pool
:
self
.
token_to_kv_pool
.
dec_refs
(
indices
)
else
:
return
torch
.
tensor
([],
dtype
=
torch
.
int64
),
self
.
root_node
# Radix Cache takes one ref in memory pool
# Radix Cache takes one ref in memory pool
self
.
token_to_kv_pool
.
dec_refs
(
indices
[
last_uncached_pos
:
new_prefix_len
])
self
.
token_to_kv_pool
.
dec_refs
(
indices
[
last_uncached_pos
:
new_prefix_len
])
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment