Commit f0021a4d authored by 王敏's avatar 王敏
Browse files

修复新request kv cache移动错误

parent acdbc978
...@@ -531,6 +531,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase): ...@@ -531,6 +531,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
not called, meaning that the kv-cache in proposer for requests is not not called, meaning that the kv-cache in proposer for requests is not
updated, so they cannot enable spec decode in the rest decoding. updated, so they cannot enable spec decode in the rest decoding.
""" """
if self.tree_style_spec_decoding and self.kvcache_slot_to_be_moved is not None:
execute_model_req.kvcache_slot_to_be_moved = self.kvcache_slot_to_be_moved
self.kvcache_slot_to_be_moved = None
sampler_output = self.scorer_worker.execute_model(execute_model_req) sampler_output = self.scorer_worker.execute_model(execute_model_req)
assert len(sampler_output) == 1 assert len(sampler_output) == 1
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment