Merge pull request #182 from FDecaYed/deyuf/update_norm

Remove LoadLibrary half norm hack

apex/optimizers/fp16_optimizer.py

 import torch
 from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
-import ctypes
-
-stashed_err = None
-try:
-    lib = ctypes.cdll.LoadLibrary(None)
-    lib.THCudaHalfTensor_normall.argtypes=[ctypes.c_void_p, ctypes.c_void_p]
-    lib.THCudaHalfTensor_normall.restype = ctypes.c_float
-    def fused_norm(input):
-        if input.type() == 'torch.cuda.HalfTensor':
-            # 16384 is half 2 if you stare at it long enough
-            return lib.THCudaHalfTensor_normall(torch.cuda._state_cdata,
-                input._cdata, 16384)
-        else:
-            return input.norm()
-except TypeError as err:
-    stashed_err = err
-    def fused_norm(input):
-        raise RuntimeError("Failed to create fused_norm. This may happen on Windows "
-              "because of lib = ctypes.cdll.LoadLibrary(None):  you can't "
-              "LoadLibrary with None.  Original exception message was ",
-              stashed_err)
-
 
 class FP16_Optimizer(object):
     """
@@ -136,9 +114,13 @@ class FP16_Optimizer(object):
             Total norm of the current fp16 gradients (viewed as a single vector).
             Returns -1 if the most recently computed fp16 gradients overflowed
         """
-        # TODO: currently using pre-1.0 api, and not most efficient with copy to cpu and sync
+        # TODO: Not most efficient with copy to cpu and sync
         # only support 2-norm now
-        norm = float(fused_norm(fp16_grads_flat))
+        # for torch version <= 1.0.1, torch.norm with dtype will fail and fall back to cast
+        try:
+            norm = float(torch.norm(fp16_grads_flat, 2.0, dtype=torch.float32))
+        except TypeError as err:
+            norm = float(torch.norm(fp16_grads_flat.float(), 2.0))
         if norm == float('inf') or norm == -float('inf') or norm != norm:
             return -1
         else:
@@ -288,4 +270,3 @@ class FP16_Optimizer(object):
         # are guaranteed to exist, so we can just copy_() from the saved master params.
         for current, saved in zip(self.fp32_groups_flat, state_dict['fp32_groups_flat']):
             current.data.copy_(saved.data)
-