Skip to content

GitLab

Unverified Commit b361750a authored by huangtingwei's avatar huangtingwei Committed by GitHub
Browse files

Mooncake store get zero copy meta optimization (#9857)

parent 16e56ea6
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 0 deletions
python/sglang/srt/mem_cache/memory_pool_host.py
View file @ b361750a
......@@ -467,6 +467,7 @@ class MHATokenToKVPoolHost(HostKVCache):
ptr_list = []
key_list = []
kv_buffer_data_ptr = self.kv_buffer.data_ptr()
indices = indices.tolist()
v_offset = (
self.layer_num
* self.size
......@@ -706,6 +707,7 @@ class MLATokenToKVPoolHost(HostKVCache):
ptr_list = []
key_list = []
kv_buffer_data_ptr = self.kv_buffer.data_ptr()
indices = indices.tolist()
for index in range(0, len(indices), self.page_size):
k_ptr = (
kv_buffer_data_ptr
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment