Unverified Commit 36517f61 authored by Sayak Paul's avatar Sayak Paul Committed by GitHub
Browse files

[chore] correct qk norm list. (#10876)

correct qk norm list.
parent 64af74fc
...@@ -213,7 +213,9 @@ class Attention(nn.Module): ...@@ -213,7 +213,9 @@ class Attention(nn.Module):
self.norm_q = LpNorm(p=2, dim=-1, eps=eps) self.norm_q = LpNorm(p=2, dim=-1, eps=eps)
self.norm_k = LpNorm(p=2, dim=-1, eps=eps) self.norm_k = LpNorm(p=2, dim=-1, eps=eps)
else: else:
raise ValueError(f"unknown qk_norm: {qk_norm}. Should be None,'layer_norm','fp32_layer_norm','rms_norm'") raise ValueError(
f"unknown qk_norm: {qk_norm}. Should be one of None, 'layer_norm', 'fp32_layer_norm', 'layer_norm_across_heads', 'rms_norm', 'rms_norm_across_heads', 'l2'."
)
if cross_attention_norm is None: if cross_attention_norm is None:
self.norm_cross = None self.norm_cross = None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment