Unverified Commit e78821b4 authored by Michael Goin's avatar Michael Goin Committed by GitHub
Browse files

[Deprecation] Deprecate `--calculate-kv-scales` option (#37201)


Signed-off-by: default avatarmgoin <mgoin64@gmail.com>
Signed-off-by: default avatarMichael Goin <mgoin64@gmail.com>
parent 51f0acda
...@@ -83,7 +83,8 @@ class CacheConfig: ...@@ -83,7 +83,8 @@ class CacheConfig:
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for - "xxhash_cbor" combines canonical CBOR serialization with xxHash for
reproducible hashing. Requires the optional ``xxhash`` package.""" reproducible hashing. Requires the optional ``xxhash`` package."""
calculate_kv_scales: bool = False calculate_kv_scales: bool = False
"""This enables dynamic calculation of `k_scale` and `v_scale` when """Deprecated: This option is deprecated and will be removed in v0.19.
It enables dynamic calculation of `k_scale` and `v_scale` when
kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model
checkpoint if available. Otherwise, the scales will default to 1.0.""" checkpoint if available. Otherwise, the scales will default to 1.0."""
cpu_kvcache_space_bytes: int | None = None cpu_kvcache_space_bytes: int | None = None
...@@ -205,6 +206,18 @@ class CacheConfig: ...@@ -205,6 +206,18 @@ class CacheConfig:
object.__setattr__(self, "user_specified_block_size", True) object.__setattr__(self, "user_specified_block_size", True)
return self return self
@field_validator("calculate_kv_scales", mode="after")
@classmethod
def _warn_deprecated_calculate_kv_scales(cls, calculate_kv_scales: bool) -> bool:
if calculate_kv_scales:
logger.warning(
"The `--calculate-kv-scales` option is deprecated and will "
"be removed in v0.19. The scales will be loaded from the "
"model checkpoint if available, otherwise they default to "
"1.0."
)
return calculate_kv_scales
@field_validator("cache_dtype", mode="after") @field_validator("cache_dtype", mode="after")
@classmethod @classmethod
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType: def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment