"git@developer.sourcefind.cn:renzhc/diffusers_dcu.git" did not exist on "7b4e049eb00154219c025d20e2273f766c3bfc5f"
Unverified Commit 6412c5e4 authored by Conghui Tan's avatar Conghui Tan Committed by GitHub
Browse files

Avoid duplicated request ids in batch APIs (#4026)


Co-authored-by: default avatarconghuitan <conghuitan@tencent.com>
parent 0c020860
...@@ -284,11 +284,11 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe ...@@ -284,11 +284,11 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
file_request_list = [] file_request_list = []
all_requests = [] all_requests = []
request_ids = [] request_ids = []
for line in lines: for line_id, line in enumerate(lines):
request_data = json.loads(line) request_data = json.loads(line)
file_request_list.append(request_data) file_request_list.append(request_data)
body = request_data["body"] body = request_data["body"]
request_ids.append(request_data["custom_id"]) request_ids.append(f"{batch_id}-req_{line_id}")
# Although streaming is supported for standalone completions, it is not supported in # Although streaming is supported for standalone completions, it is not supported in
# batch mode (multiple completions in single request). # batch mode (multiple completions in single request).
...@@ -438,15 +438,9 @@ async def cancel_batch(tokenizer_manager, batch_id: str, input_file_id: str): ...@@ -438,15 +438,9 @@ async def cancel_batch(tokenizer_manager, batch_id: str, input_file_id: str):
with open(input_file_path, "r", encoding="utf-8") as f: with open(input_file_path, "r", encoding="utf-8") as f:
lines = f.readlines() lines = f.readlines()
file_request_list = []
request_ids = []
for line in lines:
request_data = json.loads(line)
file_request_list.append(request_data)
request_ids.append(request_data["custom_id"])
# Cancel requests by request_ids # Cancel requests by request_ids
for rid in request_ids: for line_id in range(len(lines)):
rid = f"{batch_id}-req_{line_id}"
tokenizer_manager.abort_request(rid=rid) tokenizer_manager.abort_request(rid=rid)
retrieve_batch = batch_storage[batch_id] retrieve_batch = batch_storage[batch_id]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment