Fix torch.cuda.MemPool() internal assertion failure (#4687)

Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>

python/pyproject.toml

@@ -74,7 +74,7 @@ srt_cpu = ["sglang[runtime_common]", "outlines>=0.0.44,<=0.1.11", "torch"]
 openai = ["openai>=1.0", "tiktoken"]
 anthropic = ["anthropic>=0.20.0"]
 litellm = ["litellm>=1.0.0"]
-torch_memory_saver = ["torch_memory_saver>=0.0.3"]
+torch_memory_saver = ["torch_memory_saver>=0.0.4"]
 test = [
     "jsonlines",
     "matplotlib",