Add qwen2 tie word embedding (#630)

fe3be159 · ylying · GitHub · 0aa189f1 · fe3be159
Unverified Commit fe3be159 authored Jul 17, 2024 by ylying Committed by GitHub Jul 16, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

python/sglang/srt/models/qwen2.py python/sglang/srt/models/qwen2.py +2 -0

No files found.
--- a/python/sglang/srt/models/qwen2.py
+++ b/python/sglang/srt/models/qwen2.py
@@ -313,6 +313,8 @@ class Qwen2ForCausalLM(nn.Module):
                param = params_dict[name]
                weight_loader = getattr(param, "weight_loader", default_weight_loader)
                weight_loader(param, loaded_weight)
+                if self.config.tie_word_embeddings and name=="model.embed_tokens.weight":
+                    weight_loader(params_dict["lm_head.weight"], loaded_weight)


 EntryClass = Qwen2ForCausalLM