Unverified Commit 37a4947d authored by Michael Goin's avatar Michael Goin Committed by GitHub
Browse files

[Bugfix] Fix layer skip logic with bitsandbytes (#9887)


Signed-off-by: default avatarmgoin <michael@neuralmagic.com>
parent 96e0c9cb
...@@ -119,7 +119,12 @@ class BitsAndBytesConfig(QuantizationConfig): ...@@ -119,7 +119,12 @@ class BitsAndBytesConfig(QuantizationConfig):
def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]): def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
return any(module_name in prefix for module_name in llm_int8_skip_modules) # Split the prefix into its dot-separated components
components = prefix.split('.')
# Check if any of the skip modules exactly matches any component
return any(module_name in components
for module_name in llm_int8_skip_modules)
class BitsAndBytesLinearMethod(LinearMethodBase): class BitsAndBytesLinearMethod(LinearMethodBase):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment