Unverified Commit 751e5ca2 authored by Lianmin Zheng's avatar Lianmin Zheng Committed by GitHub
Browse files

[minor] clean up docs and eos id (#2622)

parent 7a7ac6be
...@@ -19,4 +19,4 @@ Add unit tests under [sglang/test](https://github.com/sgl-project/sglang/tree/ma ...@@ -19,4 +19,4 @@ Add unit tests under [sglang/test](https://github.com/sgl-project/sglang/tree/ma
## For Newcomers ## For Newcomers
If you want to contribute or learn but don't have a concrete idea yet, you can pick a task labeled as "good first issue" or "help wanted" from the list below. If you want to contribute or learn but don't have a concrete idea yet, you can pick a task labeled as "good first issue" or "help wanted" from the list below.
https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22 [https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22)
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
import json import json
import logging import logging
from enum import IntEnum, auto from enum import IntEnum, auto
from functools import lru_cache
from typing import List, Optional, Set, Union from typing import List, Optional, Set, Union
import torch import torch
...@@ -48,6 +47,7 @@ class ModelConfig: ...@@ -48,6 +47,7 @@ class ModelConfig:
self.model_path = model_path self.model_path = model_path
self.revision = revision self.revision = revision
self.quantization = quantization self.quantization = quantization
# Parse args # Parse args
self.model_override_args = json.loads(model_override_args) self.model_override_args = json.loads(model_override_args)
self.hf_config = get_config( self.hf_config = get_config(
...@@ -131,6 +131,9 @@ class ModelConfig: ...@@ -131,6 +131,9 @@ class ModelConfig:
# Veirfy quantization # Veirfy quantization
self._verify_quantization() self._verify_quantization()
# Text attrs
self.hf_eos_token_id = self.get_hf_eos_token_id()
# Multimodel attrs # Multimodel attrs
self.image_token_id = getattr(self.hf_config, "image_token_id", None) self.image_token_id = getattr(self.hf_config, "image_token_id", None)
...@@ -272,7 +275,6 @@ class ModelConfig: ...@@ -272,7 +275,6 @@ class ModelConfig:
self.quantization, self.quantization,
) )
@lru_cache()
def get_hf_eos_token_id(self) -> Optional[Set[int]]: def get_hf_eos_token_id(self) -> Optional[Set[int]]:
eos_ids = getattr(self.hf_config, "eos_token_id", None) eos_ids = getattr(self.hf_config, "eos_token_id", None)
if eos_ids: if eos_ids:
......
...@@ -517,7 +517,7 @@ class Scheduler: ...@@ -517,7 +517,7 @@ class Scheduler:
stream=recv_req.stream, stream=recv_req.stream,
lora_path=recv_req.lora_path, lora_path=recv_req.lora_path,
input_embeds=recv_req.input_embeds, input_embeds=recv_req.input_embeds,
eos_token_ids=self.model_config.get_hf_eos_token_id(), eos_token_ids=self.model_config.hf_eos_token_id,
) )
req.tokenizer = self.tokenizer req.tokenizer = self.tokenizer
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment