Fix ESM models buffers (#24576)

* Fix ESM models buffers

* Remove modifs

* Tied weights keys are needed silly

* quality

src/transformers/models/esm/modeling_esm.py

@@ -96,7 +96,7 @@ class RotaryEmbedding(torch.nn.Module):
         # Generate and save the inverse frequency buffer (non trainable)
         inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))
         inv_freq = inv_freq
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.register_buffer("inv_freq", inv_freq)
 
         self._seq_len_cached = None
         self._cos_cached = None