[models] support step3v

vllm/transformers_utils/tokenizer.py

@@ -19,7 +19,7 @@ from vllm.logger import init_logger
 from vllm.lora.request import LoRARequest
 from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
                                                     TokenizerRegistry)
-from vllm.transformers_utils.tokenizers import MistralTokenizer
+from vllm.transformers_utils.tokenizers import MistralTokenizer, SentencePieceTokenizer
 from vllm.transformers_utils.utils import check_gguf_file
 from vllm.utils import make_async
 
@@ -28,6 +28,11 @@ if TYPE_CHECKING:
 
 logger = init_logger(__name__)
 
+_TOKENIZER_REGISTRY = {
+    "step1": SentencePieceTokenizer,
+    "step2": SentencePieceTokenizer
+}
+
 AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
                      TokenizerBase]
 

vllm/transformers_utils/tokenizers/__init__.py

@@ -3,10 +3,13 @@
 
 from .mistral import (MistralTokenizer, maybe_serialize_tool_calls,
                       truncate_tool_call_ids, validate_request_params)
+from vllm.transformers_utils.tokenizers.sentencepiece_tokenizer import (
+    SentencePieceTokenizer)
 from vllm.transformers_utils.tokenizers.cpm_9g import CPM9GTokenizer
 
 __all__ = [
     "MistralTokenizer", "maybe_serialize_tool_calls", "truncate_tool_call_ids",
     "validate_request_params",
-    "CPM9GTokenizer"
+    "CPM9GTokenizer",
+    "SentencePieceTokenizer"
 ]