XLNet Bug when training with apex 16-bit precision (#6567)

* xlnet fp16 bug fix

* comment cast added

* Update modeling_xlnet.py
Co-authored-by: Kevin Canwen Xu <canwenxu@126.com>

src/transformers/modeling_xlnet.py

@@ -446,7 +446,8 @@ class XLNetRelativeAttention(nn.Module):
             v_head_h = torch.einsum("ibh,hnd->ibnd", cat, self.v)
 
             # positional heads
-            k_head_r = torch.einsum("ibh,hnd->ibnd", r, self.r)
+            # type casting for fp16 support
+            k_head_r = torch.einsum("ibh,hnd->ibnd", r.type(self.r.dtype), self.r)
 
             # core attention ops
             attn_vec = self.rel_attn_core(