fix bug

804999f1 · Jiezhong Qiu · 96ed8976 · 804999f1
Commit 804999f1 authored Feb 25, 2021 by Jiezhong Qiu
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

examples/transformer-xl/mem_transformer.py examples/transformer-xl/mem_transformer.py +2 -2

No files found.
--- a/examples/transformer-xl/mem_transformer.py
+++ b/examples/transformer-xl/mem_transformer.py
@@ -380,8 +380,8 @@ from fmoe import FMoETransformerMLP
 class CustomizedMoEPositionwiseFF(FMoETransformerMLP):
    def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, moe_num_expert=64, moe_top_k=2):
        activation = nn.Sequential(
-            nn.ReLU()
+            nn.ReLU(),
-            nn.Dropout(dropout),
+            nn.Dropout(dropout)
        )
        super().__init__(num_expert=moe_num_expert, d_model=d_model, d_hidden=d_inner, top_k=moe_top_k,
                do_lnorm=True, pre_lnorm=pre_lnorm, activation=activation, dropout=dropout)