Unverified Commit a6c7ebbb authored by Liangsheng Yin's avatar Liangsheng Yin Committed by GitHub
Browse files

Add req slots leaking check (#842)

parent bb0501c0
...@@ -280,6 +280,14 @@ class ModelTpServer: ...@@ -280,6 +280,14 @@ class ModelTpServer:
"KV cache pool leak detected!" "KV cache pool leak detected!"
) )
if self.req_to_token_pool.can_use_mem_size != self.req_to_token_pool.size:
warnings.warn(
"Warning: "
f"available req slots={self.req_to_token_pool.can_use_mem_size}, "
f"total slots={self.req_to_token_pool.size}\n"
"Memory pool leak detected!"
)
def handle_generate_request( def handle_generate_request(
self, self,
recv_req: TokenizedGenerateReqInput, recv_req: TokenizedGenerateReqInput,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment