Unverified Commit 5b647543 authored by Liangsheng Yin's avatar Liangsheng Yin Committed by GitHub
Browse files

Fix the broken `--disable-radix-cache` (#451)

parent 8210ec60
...@@ -58,7 +58,7 @@ class RadixCache: ...@@ -58,7 +58,7 @@ class RadixCache:
def insert(self, key, value=None): def insert(self, key, value=None):
if self.disable: if self.disable:
return len(key) return 0
if value is None: if value is None:
value = [x for x in key] value = [x for x in key]
...@@ -76,6 +76,12 @@ class RadixCache: ...@@ -76,6 +76,12 @@ class RadixCache:
indices = self.req_to_token_pool.req_to_token[req_pool_idx, : len(token_ids)] indices = self.req_to_token_pool.req_to_token[req_pool_idx, : len(token_ids)]
new_prefix_len = self.insert(token_ids, indices.clone()) new_prefix_len = self.insert(token_ids, indices.clone())
if self.disable:
if del_in_memory_pool:
self.token_to_kv_pool.dec_refs(indices)
else:
return torch.tensor([], dtype=torch.int64), self.root_node
# Radix Cache takes one ref in memory pool # Radix Cache takes one ref in memory pool
self.token_to_kv_pool.dec_refs(indices[last_uncached_pos:new_prefix_len]) self.token_to_kv_pool.dec_refs(indices[last_uncached_pos:new_prefix_len])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment