Skip to content

GitLab

Unverified Commit 26ebb849 authored by fzyzcjy's avatar fzyzcjy Committed by GitHub
Browse files

Tiny refactor bench_serving to extract RequestFuncOutput.init_new (#6108)

parent 02973cd9
Hide whitespace changes
Inline Side-by-side
Showing with 10 additions and 8 deletions
python/sglang/bench_serving.py
View file @ 26ebb849
......@@ -73,6 +73,12 @@ class RequestFuncOutput:
error: str = ""
output_len: int = 0
@staticmethod
def init_new(request_func_input: RequestFuncInput):
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
return output
def remove_prefix(text: str, prefix: str) -> str:
return text[len(prefix) :] if text.startswith(prefix) else text
......@@ -114,8 +120,7 @@ async def async_request_trt_llm(
if args.disable_ignore_eos:
del payload["min_length"]
del payload["end_id"]
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output = RequestFuncOutput.init_new(request_func_input)
ttft = 0.0
st = time.perf_counter()
......@@ -186,8 +191,7 @@ async def async_request_openai_completions(
}
headers = get_auth_headers()
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output = RequestFuncOutput.init_new(request_func_input)
generated_text = ""
output_len = request_func_input.output_len
......@@ -269,8 +273,7 @@ async def async_request_truss(
}
headers = get_auth_headers()
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output = RequestFuncOutput.init_new(request_func_input)
generated_text = ""
ttft = 0.0
......@@ -355,8 +358,7 @@ async def async_request_sglang_generate(
headers = get_auth_headers()
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output = RequestFuncOutput.init_new(request_func_input)
generated_text = ""
output_len = request_func_input.output_len
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment