Skip to content

GitLab

Unverified Commit 5d264f4a authored by Grace Ho's avatar Grace Ho Committed by GitHub
Browse files

pass ignore_eos parameter to all benchmark_serving calls (#9349)

parent e9d517f2
Hide whitespace changes
Inline Side-by-side
Showing with 18 additions and 20 deletions
benchmarks/benchmark_serving.py
View file @ 5d264f4a
...@@ -431,16 +431,15 @@ async def benchmark( ...@@ -431,16 +431,15 @@ async def benchmark(
if profile: if profile:
print("Starting profiler...") print("Starting profiler...")
profile_input = RequestFuncInput( profile_input = RequestFuncInput(model=model_id,
model=model_id, prompt=test_prompt,
prompt=test_prompt, api_url=base_url + "/start_profile",
api_url=base_url + "/start_profile", prompt_len=test_prompt_len,
prompt_len=test_prompt_len, output_len=test_output_len,
output_len=test_output_len, logprobs=logprobs,
logprobs=logprobs, best_of=best_of,
best_of=best_of, multi_modal_content=test_mm_content,
multi_modal_content=test_mm_content, ignore_eos=ignore_eos)
)
profile_output = await request_func(request_func_input=profile_input) profile_output = await request_func(request_func_input=profile_input)
if profile_output.success: if profile_output.success:
print("Profiler started") print("Profiler started")
...@@ -453,16 +452,15 @@ async def benchmark( ...@@ -453,16 +452,15 @@ async def benchmark(
tasks: List[asyncio.Task] = [] tasks: List[asyncio.Task] = []
async for request in get_request(input_requests, request_rate): async for request in get_request(input_requests, request_rate):
prompt, prompt_len, output_len, mm_content = request prompt, prompt_len, output_len, mm_content = request
request_func_input = RequestFuncInput( request_func_input = RequestFuncInput(model=model_id,
model=model_id, prompt=prompt,
prompt=prompt, api_url=api_url,
api_url=api_url, prompt_len=prompt_len,
prompt_len=prompt_len, output_len=output_len,
output_len=output_len, logprobs=logprobs,
logprobs=logprobs, best_of=best_of,
best_of=best_of, multi_modal_content=mm_content,
multi_modal_content=mm_content, ignore_eos=ignore_eos)
)
tasks.append( tasks.append(
asyncio.create_task( asyncio.create_task(
request_func(request_func_input=request_func_input, request_func(request_func_input=request_func_input,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment