Skip to content

GitLab

Unverified Commit 3f0a3611 authored by Hailey Schoelkopf's avatar Hailey Schoelkopf Committed by GitHub
Browse files

generalize qwen pad token fix (#1146)

parent 42730d90
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
lm_eval/models/huggingface.py
View file @ 3f0a3611
...@@ -238,7 +238,7 @@ class HFLM(LM): ...@@ -238,7 +238,7 @@ class HFLM(LM):
elif self.tokenizer.eos_token: elif self.tokenizer.eos_token:
self.tokenizer.pad_token_id = self.tokenizer.eos_token_id self.tokenizer.pad_token_id = self.tokenizer.eos_token_id
else: else:
if "Qwen" in pretrained: if self.config.model_type == "qwen":
# Qwen's trust_remote_code tokenizer does not allow for adding special tokens # Qwen's trust_remote_code tokenizer does not allow for adding special tokens
self.tokenizer.pad_token = "<|endoftext|>" self.tokenizer.pad_token = "<|endoftext|>"
else: else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment