Skip to content

GitLab

"git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "e1c3ac25515839146c93427e55941de9cee3401e"
Unverified Commit c63bfc30 authored by Younes Belkada's avatar Younes Belkada Committed by GitHub
Browse files

[`RWKV`] Fix RWKV 4bit (#23910) 

fix RWKV 4bit
parent 55451c66
Show whitespace changes
Inline Side-by-side
Showing with 7 additions and 0 deletions
src/transformers/models/rwkv/modeling_rwkv.py
View file @ c63bfc30
...@@ -710,6 +710,13 @@ class RwkvModel(RwkvPreTrainedModel): ...@@ -710,6 +710,13 @@ class RwkvModel(RwkvPreTrainedModel):
if hasattr(block.attention.output.weight, "SCB"): if hasattr(block.attention.output.weight, "SCB"):
block.attention.output.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every)) block.attention.output.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every))
block.feed_forward.value.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every)) block.feed_forward.value.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every))
elif hasattr(block.attention.output.weight, "quant_state"):
block.attention.output.weight.quant_state[0].div_(
2 ** int(block_id // self.config.rescale_every)
)
block.feed_forward.value.weight.quant_state[0].div_(
2 ** int(block_id // self.config.rescale_every)
)
else: else:
block.attention.output.weight.div_(2 ** int(block_id // self.config.rescale_every)) block.attention.output.weight.div_(2 ** int(block_id // self.config.rescale_every))
block.feed_forward.value.weight.div_(2 ** int(block_id // self.config.rescale_every)) block.feed_forward.value.weight.div_(2 ** int(block_id // self.config.rescale_every))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment