Skip to content

GitLab

Commit 7741808b authored by Thor Johnsen's avatar Thor Johnsen
Browse files

Bug fix

parent 12458152
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
apex/contrib/optimizers/distributed_fused_lamb.py
View file @ 7741808b
...@@ -96,7 +96,7 @@ class DistributedFusedLAMB(torch.optim.Optimizer): ...@@ -96,7 +96,7 @@ class DistributedFusedLAMB(torch.optim.Optimizer):
import amp_C import amp_C
self.multi_tensor_l2norm = amp_C.multi_tensor_l2norm self.multi_tensor_l2norm = amp_C.multi_tensor_l2norm
self.adam_w_mode = 1 if adam_w_mode else 0 self._adam_w_mode = 1 if adam_w_mode else 0
self._use_nvlamb = use_nvlamb self._use_nvlamb = use_nvlamb
self._is_accumulation_step = False self._is_accumulation_step = False
self._last_step = False self._last_step = False
...@@ -465,7 +465,7 @@ class DistributedFusedLAMB(torch.optim.Optimizer): ...@@ -465,7 +465,7 @@ class DistributedFusedLAMB(torch.optim.Optimizer):
self._contrib_bias_correction, self._contrib_bias_correction,
self._param_state['step']+1, self._param_state['step']+1,
self._contrib_epsilon, self._contrib_epsilon,
1, # adam mode. FIXME: Correct value self._adam_w_mode,
self._contrib_weight_decay, self._contrib_weight_decay,
self.L2_grad_norm, self.L2_grad_norm,
max_grad_norm) max_grad_norm)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment