Unverified Commit 46b4a027 authored by will b.'s avatar will b. Committed by GitHub
Browse files

Fix DeepSeek V2 RoPE initialization error (#33501)


Signed-off-by: default avatarEduardo Salinas <edus@microsoft.com>
Signed-off-by: default avatarcatswe <212922539+catswe@users.noreply.github.com>
Co-authored-by: default avatarEduardo Salinas <edus@microsoft.com>
parent 8869cd8e
...@@ -75,7 +75,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase): ...@@ -75,7 +75,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
self.rotary_dim, self.rotary_dim,
2, 2,
dtype=torch.float, dtype=torch.float,
device=current_platform.device_type,
) )
/ self.rotary_dim / self.rotary_dim
) )
...@@ -104,7 +103,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase): ...@@ -104,7 +103,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
inv_freq = self._compute_inv_freq(self.scaling_factor) inv_freq = self._compute_inv_freq(self.scaling_factor)
t = torch.arange( t = torch.arange(
self.max_position_embeddings * self.scaling_factor, self.max_position_embeddings * self.scaling_factor,
device=current_platform.device_type,
dtype=torch.float32, dtype=torch.float32,
) )
freqs = torch.einsum("i,j -> ij", t, inv_freq) freqs = torch.einsum("i,j -> ij", t, inv_freq)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment