Fix `torchscript` tests for `AltCLIP` (#21102)

fix torchscript tests for AltCLIP
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/altclip/test_modeling_altclip.py

@@ -490,6 +490,15 @@ class AltCLIPModelTest(ModelTesterMixin, unittest.TestCase):
             model_state_dict = model.state_dict()
             loaded_model_state_dict = loaded_model.state_dict()
 
+            non_persistent_buffers = {}
+            for key in loaded_model_state_dict.keys():
+                if key not in model_state_dict.keys():
+                    non_persistent_buffers[key] = loaded_model_state_dict[key]
+
+            loaded_model_state_dict = {
+                key: value for key, value in loaded_model_state_dict.items() if key not in non_persistent_buffers
+            }
+
             self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys()))
 
             models_equal = True