Unverified Commit 3f562547 authored by moto's avatar moto Committed by GitHub
Browse files

Replace dropout with Dropout (#1815)

parent 02def7c4
......@@ -225,7 +225,7 @@ class SelfAttention(Module):
self.embed_dim = embed_dim
self.num_heads = num_heads
self.dropout = dropout
self.dropout = torch.nn.Dropout(dropout)
self.head_dim = head_dim
self.scaling = self.head_dim ** -0.5
......@@ -273,7 +273,7 @@ class SelfAttention(Module):
weights += attention_mask
weights = torch.nn.functional.softmax(weights, dim=-1)
weights = torch.nn.functional.dropout(weights, p=self.dropout, training=self.training)
weights = self.dropout(weights)
output = weights @ v # B, nH, L, Hd
output = output.transpose(2, 1).reshape(batch_size, length, embed_dim)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment