Patch GPU failures (#6281)

* Pin to 1.5.0

* Patch XLM GPU test

.github/workflows/self-push.yml

@@ -36,7 +36,7 @@ jobs:
       run: |
         source .env/bin/activate
         pip install --upgrade pip
-        pip install torch --no-cache-dir
+        pip install torch!=1.6.0 --no-cache-dir
         pip install .[sklearn,testing]
 
     - name: Are GPUs recognized by our DL frameworks

src/transformers/modeling_xlm.py

@@ -496,11 +496,13 @@ class XLMModel(XLMPreTrainedModel):
         else:
             bs, slen = inputs_embeds.size()[:-1]
 
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
         if lengths is None:
             if input_ids is not None:
                 lengths = (input_ids != self.pad_index).sum(dim=1).long()
             else:
-                lengths = torch.LongTensor([slen] * bs)
+                lengths = torch.tensor([slen] * bs, device=device)
         # mask = input_ids != self.pad_index
 
         # check inputs