Unverified Commit c250939e authored by ybyang's avatar ybyang Committed by GitHub
Browse files

[Fix Chat API] add request id for chat/completion for tracing (#6364)

parent b6909aa2
...@@ -1193,6 +1193,7 @@ def v1_chat_generate_request( ...@@ -1193,6 +1193,7 @@ def v1_chat_generate_request(
top_logprobs_nums = top_logprobs_nums[0] top_logprobs_nums = top_logprobs_nums[0]
modalities_list = modalities_list[0] modalities_list = modalities_list[0]
lora_paths = lora_paths[0] lora_paths = lora_paths[0]
request_ids = request_ids[0]
else: else:
if tokenizer_manager.model_config.is_multimodal: if tokenizer_manager.model_config.is_multimodal:
# processor will need text input # processor will need text input
...@@ -1429,7 +1430,9 @@ async def v1_chat_completions( ...@@ -1429,7 +1430,9 @@ async def v1_chat_completions(
return create_error_response("Invalid request body, error: ", str(e)) return create_error_response("Invalid request body, error: ", str(e))
all_requests = [ChatCompletionRequest(**request_json)] all_requests = [ChatCompletionRequest(**request_json)]
created = int(time.time()) created = int(time.time())
adapted_request, request = v1_chat_generate_request(all_requests, tokenizer_manager) adapted_request, request = v1_chat_generate_request(
all_requests, tokenizer_manager, request_ids=[all_requests[0].rid]
)
if adapted_request.stream: if adapted_request.stream:
parser_dict = {} parser_dict = {}
......
...@@ -392,6 +392,9 @@ class ChatCompletionRequest(BaseModel): ...@@ -392,6 +392,9 @@ class ChatCompletionRequest(BaseModel):
stream_reasoning: bool = True stream_reasoning: bool = True
chat_template_kwargs: Optional[Dict] = None chat_template_kwargs: Optional[Dict] = None
# The request id.
rid: Optional[Union[List[str], str]] = None
# For PD disaggregation # For PD disaggregation
bootstrap_host: Optional[str] = None bootstrap_host: Optional[str] = None
bootstrap_port: Optional[int] = None bootstrap_port: Optional[int] = None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment