Skip to content

GitLab

Unverified Commit 616b59f3 authored by rainred's avatar rainred Committed by GitHub
Browse files

[Feature] modify Runtime to support skip_tokenizer_init (#1088) 


Co-authored-by: default avatarlzhang <zhanglei@modelbest.cn>
parent c8423ca3
Hide whitespace changes
Inline Side-by-side
Showing with 19 additions and 9 deletions
python/sglang/srt/server.py
View file @ 616b59f3
...@@ -533,11 +533,18 @@ class Runtime: ...@@ -533,11 +533,18 @@ class Runtime:
prompt: str, prompt: str,
sampling_params: Optional[Dict] = None, sampling_params: Optional[Dict] = None,
): ):
json_data = { if self.server_args.skip_tokenizer_init:
"text": prompt, json_data = {
"sampling_params": sampling_params, "input_ids": prompt,
"stream": True, "sampling_params": sampling_params,
} "stream": True,
}
else:
json_data = {
"text": prompt,
"sampling_params": sampling_params,
"stream": True,
}
pos = 0 pos = 0
timeout = aiohttp.ClientTimeout(total=3 * 3600) timeout = aiohttp.ClientTimeout(total=3 * 3600)
...@@ -549,10 +556,13 @@ class Runtime: ...@@ -549,10 +556,13 @@ class Runtime:
if chunk == "data: [DONE]\n\n": if chunk == "data: [DONE]\n\n":
break break
data = json.loads(chunk[5:].strip("\n")) data = json.loads(chunk[5:].strip("\n"))
cur = data["text"][pos:] if hasattr(data, "text"):
if cur: cur = data["text"][pos:]
yield cur if cur:
pos += len(cur) yield cur
pos += len(cur)
else:
yield data
add_request = async_generate add_request = async_generate
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment