Skip to content

GitLab

Unverified Commit aaab103b authored by Casper's avatar Casper Committed by GitHub
Browse files

Merge pull request #89 from casper-hansen/falcon_fix_kv_heads 

Fix Falcon n_kv_heads parameter
parents a4ea423d 80eea43e
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
awq/modules/fused/block.py
View file @ aaab103b
...@@ -38,7 +38,7 @@ class FalconDecoderLayer(nn.Module): ...@@ -38,7 +38,7 @@ class FalconDecoderLayer(nn.Module):
input_layernorm=None, ln_attn=None, ln_mlp=None, new_decoder_arch=True): input_layernorm=None, ln_attn=None, ln_mlp=None, new_decoder_arch=True):
super().__init__() super().__init__()
self.n_heads = n_heads self.n_heads = n_heads
self.n_kv_heads = 8 self.n_kv_heads = 8 if new_decoder_arch else 0
self.hidden_size = hidden_size self.hidden_size = hidden_size
self.new_decoder_arch = new_decoder_arch self.new_decoder_arch = new_decoder_arch
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment