clean up apex integration

pytorch_pretrained_bert/convert_tf_checkpoint_to_pytorch.py

@@ -59,9 +59,9 @@ def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytor
                 l = re.split(r'_(\d+)', m_name)
             else:
                 l = [m_name]
-            if l[0] == 'kernel':
+            if l[0] == 'kernel' or l[0] == 'gamma':
                 pointer = getattr(pointer, 'weight')
-            elif l[0] == 'output_bias':
+            elif l[0] == 'output_bias' or l[0] == 'beta':
                 pointer = getattr(pointer, 'bias')
             elif l[0] == 'output_weights':
                 pointer = getattr(pointer, 'weight')

pytorch_pretrained_bert/modeling.py

@@ -516,9 +516,9 @@ class PreTrainedBertModel(nn.Module):
         for key in state_dict.keys():
             new_key = None
             if 'gamma' in key:
-                new_key = key.replace('gamma','weight')
+                new_key = key.replace('gamma', 'weight')
             if 'beta' in key:
-                new_key = key.replace('beta','bias')
+                new_key = key.replace('beta', 'bias')
             if new_key:
                 old_keys.append(key)
                 new_keys.append(new_key)

tests/optimization_test.py

@@ -35,7 +35,7 @@ class OptimizationTest(unittest.TestCase):
         criterion = torch.nn.MSELoss()
         # No warmup, constant schedule, no gradient clipping
         optimizer = BertAdam(params=[w], lr=2e-1,
-                                          weight_decay=0.0,
+                                          weight_decay_rate=0.0,
                                           max_grad_norm=-1)
         for _ in range(100):
             loss = criterion(w, target)