Directly decay weight instead of L2 penalty (#157)

See https://arxiv.org/pdf/1711.05101.pdf

fairseq/nag.py

@@ -35,15 +35,14 @@ class NAG(Optimizer):
                     continue
 
                 d_p = p.grad.data
-                if weight_decay != 0:
-                    d_p.add_(weight_decay, p.data)
-
                 param_state = self.state[p]
                 if 'momentum_buffer' not in param_state:
                     param_state['momentum_buffer'] = d_p.clone().zero_()
 
                 buf = param_state['momentum_buffer']
 
+                if weight_decay != 0:
+                    p.data.mul_(1 - weight_decay)
                 p.data.add_(momentum * momentum, buf)
                 p.data.add_(-(1 + momentum) * lr, d_p)