don't convert to float bn with affine=False

apex/fp16_utils/fp16util.py

@@ -27,7 +27,7 @@ def BN_convert_float(module):
     fn to all modules, parameters, and buffers. Thus we wouldn't
     be able to guard the float conversion based on the module type.
     '''
-    if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):
+    if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True:
         module.float()
     for child in module.children():
         BN_convert_float(child)