Unverified Commit 51cd22ce authored by dancingpipi's avatar dancingpipi Committed by GitHub
Browse files

set&get llm internal tokenizer instead of the TokenizerGroup (#2741)


Co-authored-by: default avatarshujunhua1 <shujunhua1@jd.com>
parent 5ed704ec
......@@ -111,13 +111,13 @@ class LLM:
def get_tokenizer(
self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
return self.llm_engine.tokenizer
return self.llm_engine.tokenizer.tokenizer
def set_tokenizer(
self,
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
) -> None:
self.llm_engine.tokenizer = tokenizer
self.llm_engine.tokenizer.tokenizer = tokenizer
def generate(
self,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment