Skip to content

GitLab

Unverified Commit 5d264f4a authored by Grace Ho's avatar Grace Ho Committed by GitHub
Browse files

pass ignore_eos parameter to all benchmark_serving calls (#9349)

parent e9d517f2
Show whitespace changes
Inline Side-by-side
Showing with 18 additions and 20 deletions
benchmarks/benchmark_serving.py
View file @ 5d264f4a
......@@ -431,8 +431,7 @@ async def benchmark(
if profile:
print("Starting profiler...")
profile_input = RequestFuncInput(
model=model_id,
profile_input = RequestFuncInput(model=model_id,
prompt=test_prompt,
api_url=base_url + "/start_profile",
prompt_len=test_prompt_len,
......@@ -440,7 +439,7 @@ async def benchmark(
logprobs=logprobs,
best_of=best_of,
multi_modal_content=test_mm_content,
)
ignore_eos=ignore_eos)
profile_output = await request_func(request_func_input=profile_input)
if profile_output.success:
print("Profiler started")
......@@ -453,8 +452,7 @@ async def benchmark(
tasks: List[asyncio.Task] = []
async for request in get_request(input_requests, request_rate):
prompt, prompt_len, output_len, mm_content = request
request_func_input = RequestFuncInput(
model=model_id,
request_func_input = RequestFuncInput(model=model_id,
prompt=prompt,
api_url=api_url,
prompt_len=prompt_len,
......@@ -462,7 +460,7 @@ async def benchmark(
logprobs=logprobs,
best_of=best_of,
multi_modal_content=mm_content,
)
ignore_eos=ignore_eos)
tasks.append(
asyncio.create_task(
request_func(request_func_input=request_func_input,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment