Unverified Commit 01857fab authored by Ziqi Fan's avatar Ziqi Fan Committed by GitHub
Browse files

fix: update HostKVCache init to report correct msg when available memory is not enough (#8102)

parent 519ff5c8
...@@ -71,11 +71,12 @@ class HostKVCache(abc.ABC): ...@@ -71,11 +71,12 @@ class HostKVCache(abc.ABC):
requested_bytes = self.size * self.size_per_token requested_bytes = self.size * self.size_per_token
# preserve at least 10GB for other usage # preserve at least 10GB for other usage
ten_gb = 10 * (1024**3) ten_gb = 10 * (1024**3)
if requested_bytes > host_mem.available - ten_gb: available_bytes = host_mem.available - ten_gb
if requested_bytes > available_bytes:
raise ValueError( raise ValueError(
f"Not enough host memory available. Requesting " f"Not enough host memory available. Requesting "
f"{requested_bytes / 1e9:.2f} GB but only have " f"{requested_bytes / 1e9:.2f} GB but only have "
f"{host_mem.available / 1e9:.2f} GB free. Please reduce the " f"{available_bytes / 1e9:.2f} GB free. Please reduce the "
f"size of the hierarchical cache." f"size of the hierarchical cache."
) )
else: else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment