Unverified Commit 4be2255c authored by Teruaki Ishizaki's avatar Teruaki Ishizaki Committed by GitHub
Browse files

[Bugfix][Benchmarks] Fix a benchmark of deepspeed-mii backend to use api_key (#17291)


Signed-off-by: default avatarTeruaki Ishizaki <teruaki.ishizaki@ntt.com>
parent ed5d4082
...@@ -194,6 +194,11 @@ async def async_request_deepspeed_mii( ...@@ -194,6 +194,11 @@ async def async_request_deepspeed_mii(
request_func_input: RequestFuncInput, request_func_input: RequestFuncInput,
pbar: Optional[tqdm] = None, pbar: Optional[tqdm] = None,
) -> RequestFuncOutput: ) -> RequestFuncOutput:
api_url = request_func_input.api_url
assert api_url.endswith(("completions", "profile")), (
"OpenAI Completions API URL must end with 'completions' or 'profile'."
)
async with aiohttp.ClientSession( async with aiohttp.ClientSession(
trust_env=True, timeout=AIOHTTP_TIMEOUT trust_env=True, timeout=AIOHTTP_TIMEOUT
) as session: ) as session:
...@@ -204,6 +209,8 @@ async def async_request_deepspeed_mii( ...@@ -204,6 +209,8 @@ async def async_request_deepspeed_mii(
"temperature": 0.01, # deepspeed-mii does not accept 0.0 temp. "temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.
"top_p": 1.0, "top_p": 1.0,
} }
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
output = RequestFuncOutput() output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len output.prompt_len = request_func_input.prompt_len
...@@ -215,7 +222,7 @@ async def async_request_deepspeed_mii( ...@@ -215,7 +222,7 @@ async def async_request_deepspeed_mii(
st = time.perf_counter() st = time.perf_counter()
try: try:
async with session.post( async with session.post(
url=request_func_input.api_url, json=payload url=api_url, json=payload, headers=headers
) as response: ) as response:
if response.status == 200: if response.status == 200:
parsed_resp = await response.json() parsed_resp = await response.json()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment