Create the return value on device to avoid unnecessary copying from CPU (#26151)

src/transformers/models/switch_transformers/modeling_switch_transformers.py

@@ -779,7 +779,7 @@ class SwitchTransformersBlock(nn.Module):
         if isinstance(hidden_states, tuple):
             hidden_states, router_tuple = hidden_states
         else:
-            router_tuple = (torch.tensor([0], device=hidden_states.device),)
+            router_tuple = (torch.zeros((1,), device=hidden_states.device, dtype=torch.int64),)
 
         # clamp inf values to enable fp16 training
         if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():