send model to the correct device (#18800)

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py

@@ -403,6 +403,7 @@ class EncoderDecoderMixin:
         )
 
         model = SpeechEncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
+        model.to(torch_device)
         model.train()
         model.gradient_checkpointing_enable()
         model.config.decoder_start_token_id = 0

tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py

@@ -331,6 +331,7 @@ class EncoderDecoderMixin:
         )
 
         model = VisionEncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
+        model.to(torch_device)
         model.train()
         model.gradient_checkpointing_enable()
         model.config.decoder_start_token_id = 0