Skip to content

GitLab

Unverified Commit 61b63da7 authored by Yifei Zhang's avatar Yifei Zhang Committed by GitHub
Browse files

Configure the pad tokens for Qwen when using vLLM (#2810)

parent 7158f4f4
Hide whitespace changes
Inline Side-by-side
Showing with 5 additions and 5 deletions
lm_eval/models/vllm_causallms.py
View file @ 61b63da7
...@@ -113,11 +113,11 @@ class VLLM(TemplateLM): ...@@ -113,11 +113,11 @@ class VLLM(TemplateLM):
self.batch_size = "auto" self.batch_size = "auto"
eval_logger.info("Manual batching is not compatible with data parallelism.") eval_logger.info("Manual batching is not compatible with data parallelism.")
from transformers import AutoConfig from transformers import AutoConfig
self._config = AutoConfig.from_pretrained( self._config = AutoConfig.from_pretrained(
pretrained, trust_remote_code=trust_remote_code, revision=revision pretrained, trust_remote_code=trust_remote_code, revision=revision
) )
self.tokenizer = get_tokenizer( self.tokenizer = get_tokenizer(
tokenizer if tokenizer else pretrained, tokenizer if tokenizer else pretrained,
tokenizer_mode=tokenizer_mode, tokenizer_mode=tokenizer_mode,
...@@ -125,7 +125,7 @@ class VLLM(TemplateLM): ...@@ -125,7 +125,7 @@ class VLLM(TemplateLM):
revision=tokenizer_revision, revision=tokenizer_revision,
add_bos_token=add_bos_token, add_bos_token=add_bos_token,
) )
self.tokenizer = configure_pad_token(self.tokenizer) self.tokenizer = configure_pad_token(self.tokenizer, model_config=self._config)
self.add_bos_token = add_bos_token self.add_bos_token = add_bos_token
if "gemma" in pretrained.lower(): if "gemma" in pretrained.lower():
self.add_bos_token = True self.add_bos_token = True
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment