Fix chunked prefill (#984)

6ed4e3b8 · Liangsheng Yin · GitHub · 00023d62 · 6ed4e3b8
Unverified Commit 6ed4e3b8 authored Aug 07, 2024 by Liangsheng Yin Committed by GitHub Aug 07, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 1 deletion

python/sglang/srt/mem_cache/chunk_cache.py python/sglang/srt/mem_cache/chunk_cache.py +2 -1

No files found.
--- a/python/sglang/srt/mem_cache/chunk_cache.py
+++ b/python/sglang/srt/mem_cache/chunk_cache.py
@@ -56,7 +56,8 @@ class ChunkCache(BasePrefixCache):

        entry = self.entries[req.rid]
        entry.value = kv_indices
-        return kv_indices, entry
+        req.prefix_indices = kv_indices
+        req.last_node = entry

    def insert(self):
        raise NotImplementedError