Merge pull request #1154 from ziliwang/master

fix: hard coding for max number

pytorch_transformers/modeling_xlnet.py

@@ -418,6 +418,9 @@ class XLNetRelativeAttention(nn.Module):
         attn_score = (ac + bd + ef) * self.scale
         if attn_mask is not None:
             # attn_score = attn_score * (1 - attn_mask) - 1e30 * attn_mask
+            if attn_mask.dtype == torch.float16:
+                attn_score = attn_score - 65500 * attn_mask
+            else:
                 attn_score = attn_score - 1e30 * attn_mask
 
         # attention probability