"git@developer.sourcefind.cn:change/sglang.git" did not exist on "6e313c1b8bb05b2cd82a24d03824a8a5d3b570cb"
Unverified Commit 3f562547 authored by moto's avatar moto Committed by GitHub
Browse files

Replace dropout with Dropout (#1815)

parent 02def7c4
...@@ -225,7 +225,7 @@ class SelfAttention(Module): ...@@ -225,7 +225,7 @@ class SelfAttention(Module):
self.embed_dim = embed_dim self.embed_dim = embed_dim
self.num_heads = num_heads self.num_heads = num_heads
self.dropout = dropout self.dropout = torch.nn.Dropout(dropout)
self.head_dim = head_dim self.head_dim = head_dim
self.scaling = self.head_dim ** -0.5 self.scaling = self.head_dim ** -0.5
...@@ -273,7 +273,7 @@ class SelfAttention(Module): ...@@ -273,7 +273,7 @@ class SelfAttention(Module):
weights += attention_mask weights += attention_mask
weights = torch.nn.functional.softmax(weights, dim=-1) weights = torch.nn.functional.softmax(weights, dim=-1)
weights = torch.nn.functional.dropout(weights, p=self.dropout, training=self.training) weights = self.dropout(weights)
output = weights @ v # B, nH, L, Hd output = weights @ v # B, nH, L, Hd
output = output.transpose(2, 1).reshape(batch_size, length, embed_dim) output = output.transpose(2, 1).reshape(batch_size, length, embed_dim)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment