removed optimized l2 grad clipping

0b6bdca6 · Mohammad · 4ff2c963 · 0b6bdca6
Commit 0b6bdca6 authored Apr 21, 2020 by Mohammad
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

megatron/mpu/grads.py megatron/mpu/grads.py +2 -2

No files found.
--- a/megatron/mpu/grads.py
+++ b/megatron/mpu/grads.py
@@ -101,8 +101,8 @@ def clip_grad_norm(parameters, max_norm, norm_type=2):
        if clip_coef < 1:
            for p in parameters:
                p.grad.data.mul_(clip_coef)
-    elif norm_type == 2:
+    #elif norm_type == 2:
-        total_norm = l2_grad_clipper(parameters, max_norm)
+    #    total_norm = l2_grad_clipper(parameters, max_norm)
    else:
        total_norm = 0