serving_responses.py 85 KB
Newer Older
1
2
3
4
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import asyncio
5
import json
6
import time
7
import uuid
8
from collections import deque
9
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
10
from contextlib import AsyncExitStack
11
from copy import copy
12
from http import HTTPStatus
13
from typing import Final
14
15
16

import jinja2
from fastapi import Request
17
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
    ResponseCodeInterpreterCallInterpretingEvent,
23
24
25
    ResponseCodeInterpreterToolCallParam,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
26
27
    ResponseFunctionCallArgumentsDeltaEvent,
    ResponseFunctionCallArgumentsDoneEvent,
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    ResponseFunctionToolCall,
    ResponseFunctionWebSearch,
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponseOutputMessage,
    ResponseOutputText,
    ResponseReasoningItem,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseTextDeltaEvent,
    ResponseTextDoneEvent,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
    response_function_web_search,
    response_text_delta_event,
)
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
48
from openai.types.responses.response_reasoning_item import (
49
50
    Content as ResponseReasoningTextContent,
)
51
from openai.types.responses.tool import Tool
52
from openai_harmony import Message as OpenAIHarmonyMessage
53

54
from vllm import envs
55
from vllm.engine.protocol import EngineClient
56
57
58
59
60
61
62
63
64
65
from vllm.entrypoints.chat_utils import (
    ChatCompletionMessageParam,
    ChatTemplateContentFormatOption,
)
from vllm.entrypoints.context import (
    ConversationContext,
    HarmonyContext,
    SimpleContext,
    StreamingHarmonyContext,
)
66
from vllm.entrypoints.harmony_utils import (
67
    construct_harmony_previous_input_messages,
68
69
70
71
72
73
74
75
76
77
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_system_message,
    get_user_message,
    has_custom_tools,
    parse_output_message,
    parse_remaining_state,
    parse_response_input,
    render_for_completion,
)
78
from vllm.entrypoints.logger import RequestLogger
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
from vllm.entrypoints.openai.protocol import (
    DeltaMessage,
    ErrorResponse,
    InputTokensDetails,
    OutputTokensDetails,
    RequestResponseMetadata,
    ResponseCompletedEvent,
    ResponseCreatedEvent,
    ResponseInProgressEvent,
    ResponseReasoningPartAddedEvent,
    ResponseReasoningPartDoneEvent,
    ResponsesRequest,
    ResponsesResponse,
    ResponseUsage,
    StreamingResponsesResponse,
)
95
96
from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
97
from vllm.entrypoints.responses_utils import construct_chat_message_with_tool_call
98
from vllm.entrypoints.tool_server import ToolServer
99
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
100
from vllm.logger import init_logger
101
102
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
103
from vllm.outputs import CompletionOutput
104
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
105
106
107
108
109
110
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.utils import random_uuid

logger = init_logger(__name__)


111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def extract_tool_types(tools: list[Tool]) -> set[str]:
    """
    Extracts the tool types from the given tools.
    """
    tool_types: set[str] = set()
    for tool in tools:
        if tool.type == "mcp":
            # Allow the MCP Tool type to enable built in tools if the
            # server_label is allowlisted in
            # envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS
            if tool.server_label in envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS:
                tool_types.add(tool.server_label)
        else:
            tool_types.add(tool.type)
    return tool_types


128
129
130
131
132
133
class OpenAIServingResponses(OpenAIServing):
    def __init__(
        self,
        engine_client: EngineClient,
        models: OpenAIServingModels,
        *,
134
135
        request_logger: RequestLogger | None,
        chat_template: str | None,
136
137
138
139
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
        reasoning_parser: str = "",
        enable_auto_tools: bool = False,
140
141
        tool_parser: str | None = None,
        tool_server: ToolServer | None = None,
142
143
        enable_prompt_tokens_details: bool = False,
        enable_force_include_usage: bool = False,
144
        enable_log_outputs: bool = False,
145
        log_error_stack: bool = False,
146
147
148
149
150
151
    ) -> None:
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
152
            log_error_stack=log_error_stack,
153
154
155
156
        )

        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
157
        self.enable_log_outputs = enable_log_outputs
158

159
160
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
161
        )
162
163
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
        self.enable_force_include_usage = enable_force_include_usage
164
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
165
166
167
        if self.default_sampling_params:
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
168
169
170
171
172
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
173

174
175
176
177
178
        # If False (default), the "store" option is (silently) ignored and the
        # response is not stored. If True, the response is stored in memory.
        # NOTE(woosuk): This may not be intuitive for users, as the default
        # behavior in OpenAI's Responses API is to store the response, but
        # vLLM's default behavior is not.
179
        self.enable_store = envs.VLLM_ENABLE_RESPONSES_API_STORE
180
181
182
183
        if self.enable_store:
            logger.warning_once(
                "`VLLM_ENABLE_RESPONSES_API_STORE` is enabled. This may "
                "cause a memory leak since we never remove responses from "
184
185
                "the store."
            )
186

187
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
188
        if self.use_harmony:
189
190
191
192
            logger.warning(
                "For gpt-oss, we ignore --enable-auto-tool-choice "
                "and always enable tool use."
            )
193
194
195
196
197
            # OpenAI models have two EOS-like tokens: <|return|> and <|call|>.
            # We need to add them to the stop token ids.
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
198
199
                get_stop_tokens_for_assistant_actions()
            )
200
        self.enable_auto_tools = enable_auto_tools
201
        # set up tool use
202
203
204
205
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
        )
        self.exclude_tools_when_tool_choice_none = False
206
        # HACK(woosuk): This is a hack. We should use a better store.
207
208
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove responses from the store.
209
210
211
212
        self.response_store: dict[str, ResponsesResponse] = {}
        self.response_store_lock = asyncio.Lock()

        # HACK(woosuk): This is a hack. We should use a better store.
213
214
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove messages from the store.
215
216
        self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}

217
218
219
        # HACK(wuhang): This is a hack. We should use a better store.
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove events from the store.
220
221
222
        self.event_store: dict[
            str, tuple[deque[StreamingResponsesResponse], asyncio.Event]
        ] = {}
223

224
225
        self.background_tasks: dict[str, asyncio.Task] = {}

226
227
        self.tool_server = tool_server

228
    def _validate_generator_input(
229
        self, engine_prompt: EngineTokensPrompt
230
    ) -> ErrorResponse | None:
231
232
233
234
235
236
        """Add validations to the input to the generator here."""
        if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
            error_message = (
                "The engine prompt length"
                f" {len(engine_prompt['prompt_token_ids'])} "
                f"exceeds the max_model_len {self.max_model_len}. "
237
238
                "Please reduce prompt."
            )
239
240
241
242
243
244
245
            return self.create_error_response(
                err_type="invalid_request_error",
                message=error_message,
                status_code=HTTPStatus.BAD_REQUEST,
            )
        return None

246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
    def _validate_create_responses_input(
        self, request: ResponsesRequest
    ) -> ErrorResponse | None:
        if self.use_harmony and request.is_include_output_logprobs():
            return self.create_error_response(
                err_type="invalid_request_error",
                message="logprobs are not supported with gpt-oss models",
                status_code=HTTPStatus.BAD_REQUEST,
            )
        if request.store and not self.enable_store and request.background:
            return self.create_error_response(
                err_type="invalid_request_error",
                message=(
                    "This vLLM engine does not support `store=True` and "
                    "therefore does not support the background mode. To "
                    "enable these features, set the environment variable "
                    "`VLLM_ENABLE_RESPONSES_API_STORE=1` when launching "
                    "the vLLM server."
                ),
                status_code=HTTPStatus.BAD_REQUEST,
            )
267
268
269
270
271
272
273
        if request.previous_input_messages and request.previous_response_id:
            return self.create_error_response(
                err_type="invalid_request_error",
                message="Only one of `previous_input_messages` and "
                "`previous_response_id` can be set.",
                status_code=HTTPStatus.BAD_REQUEST,
            )
274
275
        return None

276
277
278
    async def create_responses(
        self,
        request: ResponsesRequest,
279
280
281
282
283
284
        raw_request: Request | None = None,
    ) -> (
        AsyncGenerator[StreamingResponsesResponse, None]
        | ResponsesResponse
        | ErrorResponse
    ):
285
286
287
288
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
            logger.error("Error with model %s", error_check_ret)
            return error_check_ret
289
290
291
        maybe_validation_error = self._validate_create_responses_input(request)
        if maybe_validation_error is not None:
            return maybe_validation_error
292
293
294
295
296
297
298

        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

299
        if request.store and not self.enable_store:
300
301
302
303
304
305
306
            # Disable the store option.
            # NOTE(woosuk): Although returning an error is possible, we opted
            # to implicitly disable store and process the request anyway, as
            # we assume most users do not intend to actually store the response
            # (i.e., their request's `store=True` just because it's the default
            # value).
            request.store = False
307

308
309
310
311
312
313
314
315
316
317
318
        # Handle the previous response ID.
        prev_response_id = request.previous_response_id
        if prev_response_id is not None:
            async with self.response_store_lock:
                prev_response = self.response_store.get(prev_response_id)
            if prev_response is None:
                return self._make_not_found_error(prev_response_id)
        else:
            prev_response = None

        try:
319
            lora_request = self._maybe_get_adapters(request)
320
            model_name = self.models.model_name(lora_request)
321
            tokenizer = await self.engine_client.get_tokenizer()
322

323
324
            if self.use_harmony:
                messages, request_prompts, engine_prompts = (
325
326
                    self._make_request_with_harmony(request, prev_response)
                )
327
            else:
328
329
330
                messages, request_prompts, engine_prompts = await self._make_request(
                    request, prev_response, tokenizer
                )
331

332
333
334
335
336
337
338
        except (
            ValueError,
            TypeError,
            RuntimeError,
            jinja2.TemplateError,
            NotImplementedError,
        ) as e:
339
340
341
            logger.exception("Error in preprocessing prompt inputs")
            return self.create_error_response(f"{e} {e.__cause__}")

342
        request_metadata = RequestResponseMetadata(request_id=request.request_id)
343
344
345
346
        if raw_request:
            raw_request.state.request_metadata = request_metadata

        # Schedule the request and get the result generator.
347
        generators: list[AsyncGenerator[ConversationContext, None]] = []
348
349
350
351
352
353
354

        builtin_tool_list: list[str] = []
        if self.use_harmony and self.tool_server is not None:
            if self.tool_server.has_tool("browser"):
                builtin_tool_list.append("browser")
            if self.tool_server.has_tool("python"):
                builtin_tool_list.append("python")
355
356
            if self.tool_server.has_tool("container"):
                builtin_tool_list.append("container")
357

358
359
360
361
362
363
364
        if self.tool_server is not None:
            available_tools = builtin_tool_list
        else:
            assert len(builtin_tool_list) == 0
            available_tools = []
        try:
            for i, engine_prompt in enumerate(engine_prompts):
365
366
367
368
                maybe_error = self._validate_generator_input(engine_prompt)
                if maybe_error is not None:
                    return maybe_error

369
                default_max_tokens = self.max_model_len - len(
370
371
                    engine_prompt["prompt_token_ids"]
                )
372

373
                sampling_params = request.to_sampling_params(
374
375
                    default_max_tokens, self.default_sampling_params
                )
376

377
378
379
380
381
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
382
383
384
385

                context: ConversationContext
                if self.use_harmony:
                    if request.stream:
386
                        context = StreamingHarmonyContext(messages, available_tools)
387
388
389
390
                    else:
                        context = HarmonyContext(messages, available_tools)
                else:
                    context = SimpleContext()
391
392
393
394
395
396
397
398
399
400
401
402
403

                if self.reasoning_parser is not None:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                    if sampling_params.structured_outputs is None:
                        sampling_params.structured_outputs = StructuredOutputsParams()
                    struct_out = sampling_params.structured_outputs
                    if struct_out.all_non_structural_tag_constraints_none():
                        sampling_params.structured_outputs.structural_tag = (
                            reasoning_parser.prepare_structured_tag(
                                sampling_params.structured_outputs.structural_tag,
                                self.tool_server,
                            )
                        )
404
405
406
407
408
409
410
411
412
                generator = self._generate_with_builtin_tools(
                    request_id=request.request_id,
                    request_prompt=request_prompts[i],
                    engine_prompt=engine_prompt,
                    sampling_params=sampling_params,
                    context=context,
                    lora_request=lora_request,
                    priority=request.priority,
                    trace_headers=trace_headers,
413
                )
414
415
416
417
                generators.append(generator)
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
418

419
        assert len(generators) == 1
420
        (result_generator,) = generators
421
422
423
424

        # Store the input messages.
        if request.store:
            self.msg_store[request.request_id] = messages
425

426
427
428
429
430
431
432
433
434
435
436
437
438
        if request.background:
            created_time = int(time.time())
            response = ResponsesResponse.from_request(
                request,
                sampling_params,
                model_name=model_name,
                created_time=created_time,
                output=[],
                status="queued",
                usage=None,
            )
            async with self.response_store_lock:
                self.response_store[response.id] = response
439

440
            # Run the request in the background.
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
            if request.stream:
                task = asyncio.create_task(
                    self._run_background_request_stream(
                        request,
                        sampling_params,
                        result_generator,
                        context,
                        model_name,
                        tokenizer,
                        request_metadata,
                        created_time,
                    ),
                    name=f"create_{request.request_id}",
                )
            else:
                task = asyncio.create_task(
                    self._run_background_request(
                        request,
                        sampling_params,
                        result_generator,
                        context,
                        model_name,
                        tokenizer,
                        request_metadata,
                        created_time,
                    ),
                    name=f"create_{response.id}",
                )
469

470
471
472
473
            # For cleanup.
            response_id = response.id
            self.background_tasks[response_id] = task
            task.add_done_callback(
474
475
                lambda _: self.background_tasks.pop(response_id, None)
            )
476
477

            if request.stream:
478
                return self.responses_background_stream_generator(request.request_id)
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
            return response

        if request.stream:
            return self.responses_stream_generator(
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
            )

        try:
            return await self.responses_full_generator(
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
            )
        except Exception as e:
            return self.create_error_response(str(e))
504

505
506
507
    async def _make_request(
        self,
        request: ResponsesRequest,
508
        prev_response: ResponsesResponse | None,
509
510
        tokenizer: AnyTokenizer,
    ):
511
512
513
514
515
516
        if request.tools is None or (
            request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
        ):
            tool_dicts = None
        else:
            tool_dicts = [tool.model_dump() for tool in request.tools]
517
518
519
520
521
522
        # Construct the input messages.
        messages = self._construct_input_messages(request, prev_response)
        _, request_prompts, engine_prompts = await self._preprocess_chat(
            request,
            tokenizer,
            messages,
523
524
            tool_dicts=tool_dicts,
            tool_parser=self.tool_parser,
525
526
527
528
529
530
531
532
            chat_template=self.chat_template,
            chat_template_content_format=self.chat_template_content_format,
        )
        return messages, request_prompts, engine_prompts

    def _make_request_with_harmony(
        self,
        request: ResponsesRequest,
533
        prev_response: ResponsesResponse | None,
534
535
536
    ):
        if request.tool_choice != "auto":
            raise NotImplementedError(
537
538
539
                "Only 'auto' tool_choice is supported in response API with Harmony"
            )
        messages = self._construct_input_messages_with_harmony(request, prev_response)
540
541
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
542
543
544
545
546

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

547
548
        return messages, [prompt_token_ids], [engine_prompt]

549
550
551
552
553
554
    async def _initialize_tool_sessions(
        self,
        request: ResponsesRequest,
        context: ConversationContext,
        exit_stack: AsyncExitStack,
    ):
555
556
557
558
        # we should only initialize the tool session if the request needs tools
        if len(request.tools) == 0:
            return
        mcp_tools = {
559
            tool.server_label: tool for tool in request.tools if tool.type == "mcp"
560
        }
561
562
563
        await context.init_tool_sessions(
            self.tool_server, exit_stack, request.request_id, mcp_tools
        )
564

565
566
567
568
    async def responses_full_generator(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
569
        result_generator: AsyncIterator[ConversationContext],
570
        context: ConversationContext,
571
572
573
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
574
575
        created_time: int | None = None,
    ) -> ErrorResponse | ResponsesResponse:
576
577
578
        if created_time is None:
            created_time = int(time.time())

579
580
        async with AsyncExitStack() as exit_stack:
            try:
581
                await self._initialize_tool_sessions(request, context, exit_stack)
582
583
584
585
586
587
588
                async for _ in result_generator:
                    pass
            except asyncio.CancelledError:
                return self.create_error_response("Client disconnected")
            except ValueError as e:
                # TODO: Use a vllm-specific Validation Error
                return self.create_error_response(str(e))
589

590
591
592
593
594
        # NOTE: Implementation of stauts is still WIP, but for now
        # we guarantee that if the status is not "completed", it is accurate.
        # "completed" is implemented as the "catch-all" for now.
        status: ResponseStatus = "completed"

595
596
        input_messages = None
        output_messages = None
597
598
599
        if self.use_harmony:
            assert isinstance(context, HarmonyContext)
            output = self._make_response_output_items_with_harmony(context)
600
            if request.enable_response_messages:
601
602
                input_messages = context.messages[: context.num_init_messages]
                output_messages = context.messages[context.num_init_messages :]
603
            num_tool_output_tokens = context.num_tool_output_tokens
604
605
606
607
608
609
610
            if len(output) > 0:
                if context.finish_reason == "length":
                    status = "incomplete"
                elif context.finish_reason == "abort":
                    status = "cancelled"
            else:
                status = "incomplete"
611
        else:
612
613
614
615
616
617
            assert isinstance(context, SimpleContext)
            final_res = context.last_output
            assert final_res is not None
            assert len(final_res.outputs) == 1
            final_output = final_res.outputs[0]

618
            output = self._make_response_output_items(request, final_output, tokenizer)
619

620
621
622
623
            # TODO: context for non-gptoss models doesn't use messages
            # so we can't get them out yet
            if request.enable_response_messages:
                raise NotImplementedError(
624
625
                    "enable_response_messages is currently only supported for gpt-oss"
                )
626
627
            # Calculate usage.
            assert final_res.prompt_token_ids is not None
628
629
            num_tool_output_tokens = 0

630
631
632
633
634
        assert isinstance(context, (SimpleContext, HarmonyContext))
        num_prompt_tokens = context.num_prompt_tokens
        num_generated_tokens = context.num_output_tokens
        num_cached_tokens = context.num_cached_tokens
        num_reasoning_tokens = context.num_reasoning_tokens
635
636
637
638

        usage = ResponseUsage(
            input_tokens=num_prompt_tokens,
            output_tokens=num_generated_tokens,
639
            total_tokens=num_prompt_tokens + num_generated_tokens,
640
641
642
643
644
645
646
647
648
            input_tokens_details=InputTokensDetails(
                cached_tokens=num_cached_tokens,
                input_tokens_per_turn=[
                    turn.input_tokens for turn in context.all_turn_metrics
                ],
                cached_tokens_per_turn=[
                    turn.cached_input_tokens for turn in context.all_turn_metrics
                ],
            ),
649
            output_tokens_details=OutputTokensDetails(
650
                reasoning_tokens=num_reasoning_tokens,
651
                tool_output_tokens=num_tool_output_tokens,
652
653
654
655
656
657
                output_tokens_per_turn=[
                    turn.output_tokens for turn in context.all_turn_metrics
                ],
                tool_output_tokens_per_turn=[
                    turn.tool_output_tokens for turn in context.all_turn_metrics
                ],
658
            ),
659
660
661
662
        )
        response = ResponsesResponse.from_request(
            request,
            sampling_params,
663
664
            input_messages=input_messages,
            output_messages=output_messages,
665
666
667
            model_name=model_name,
            created_time=created_time,
            output=output,
668
            status=status,
669
670
671
672
673
674
675
            usage=usage,
        )

        if request.store:
            async with self.response_store_lock:
                stored_response = self.response_store.get(response.id)
                # If the response is already cancelled, don't update it.
676
                if stored_response is None or stored_response.status != "cancelled":
677
678
679
                    self.response_store[response.id] = response
        return response

680
681
682
683
684
685
    def _topk_logprobs(
        self,
        logprobs: dict[int, SampleLogprob],
        top_logprobs: int,
        tokenizer: AnyTokenizer,
    ) -> list[LogprobTopLogprob]:
686
687
688
689
690
        """Returns the top-k logprobs from the logprobs dictionary."""
        out = []
        for i, (token_id, _logprob) in enumerate(logprobs.items()):
            if i >= top_logprobs:
                break
691
692
693
694
695
            text = (
                _logprob.decoded_token
                if _logprob.decoded_token is not None
                else tokenizer.decode([token_id])
            )
696
697
698
699
700
            out.append(
                LogprobTopLogprob(
                    token=text,
                    logprob=max(_logprob.logprob, -9999.0),
                    bytes=list(text.encode("utf-8", errors="replace")),
701
702
                )
            )
703
704
705
        return out

    def _create_response_logprobs(
706
707
        self,
        token_ids: Sequence[int],
708
        logprobs: SampleLogprobs | None,
709
        tokenizer: AnyTokenizer,
710
        top_logprobs: int | None = None,
711
    ) -> list[Logprob]:
712
713
        assert logprobs is not None, "logprobs must be provided"
        assert len(token_ids) == len(logprobs), (
714
715
            "token_ids and logprobs.token_ids must have the same length"
        )
716
717
718
719
        out = []
        for i, token_id in enumerate(token_ids):
            logprob = logprobs[i]
            token_logprob = logprob[token_id]
720
721
722
723
724
            text = (
                token_logprob.decoded_token
                if token_logprob.decoded_token is not None
                else tokenizer.decode([token_id])
            )
725
726
727
728
729
            out.append(
                Logprob(
                    token=text,
                    logprob=max(token_logprob.logprob, -9999.0),
                    bytes=list(text.encode("utf-8", errors="replace")),
730
731
732
733
734
735
736
                    top_logprobs=(
                        self._topk_logprobs(
                            logprob, top_logprobs=top_logprobs, tokenizer=tokenizer
                        )
                        if top_logprobs
                        else []
                    ),
737
738
                )
            )
739
740
        return out

741
742
743
    def _create_stream_response_logprobs(
        self,
        token_ids: Sequence[int],
744
        logprobs: SampleLogprobs | None,
745
        tokenizer: AnyTokenizer,
746
        top_logprobs: int | None = None,
747
    ) -> list[response_text_delta_event.Logprob]:
748
749
750
751
752
753
        lgs = self._create_response_logprobs(
            token_ids=token_ids,
            logprobs=logprobs,
            tokenizer=tokenizer,
            top_logprobs=top_logprobs,
        )
754
755
756
757
758
759
        return [
            response_text_delta_event.Logprob(
                token=lg.token,
                logprob=lg.logprob,
                top_logprobs=[
                    response_text_delta_event.LogprobTopLogprob(
760
761
                        token=tl.token, logprob=tl.logprob
                    )
762
                    for tl in lg.top_logprobs
763
764
765
                ],
            )
            for lg in lgs
766
767
        ]

768
769
770
771
772
773
774
775
776
777
778
779
780
    def _make_response_output_items(
        self,
        request: ResponsesRequest,
        final_output: CompletionOutput,
        tokenizer: AnyTokenizer,
    ) -> list[ResponseOutputItem]:
        if self.reasoning_parser:
            try:
                reasoning_parser = self.reasoning_parser(tokenizer)
            except RuntimeError as e:
                logger.exception("Error in reasoning parser creation.")
                raise e

781
782
783
            reasoning_content, content = reasoning_parser.extract_reasoning_content(
                final_output.text, request=request
            )
784
785
786
787
        else:
            reasoning_content = None
            content = final_output.text

788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            output_text = ""
            if content:
                output_text = content
            elif reasoning_content:
                output_text = f"[reasoning: {reasoning_content}]"

            if output_text:
                self.request_logger.log_outputs(
                    request_id=request.request_id,
                    outputs=output_text,
                    output_token_ids=final_output.token_ids,
                    finish_reason=final_output.finish_reason,
                    is_streaming=False,
                    delta=False,
                )

806
807
        reasoning_item = None
        message_item = None
808
809
810
811
812
813
        if reasoning_content:
            reasoning_item = ResponseReasoningItem(
                id=f"rs_{random_uuid()}",
                summary=[],
                type="reasoning",
                content=[
814
815
816
                    ResponseReasoningTextContent(
                        text=reasoning_content, type="reasoning_text"
                    )
817
818
819
                ],
                status=None,  # NOTE: Only the last output item has status.
            )
820
821
822
823
824
825
826
        tool_calls, content = self._parse_tool_calls_from_content(
            request=request,
            tokenizer=tokenizer,
            content=content,
            enable_auto_tools=self.enable_auto_tools,
            tool_parser_cls=self.tool_parser,
        )
827
828
829
830
831
        if content:
            output_text = ResponseOutputText(
                text=content,
                annotations=[],  # TODO
                type="output_text",
832
833
834
835
836
837
838
839
840
841
                logprobs=(
                    self._create_response_logprobs(
                        token_ids=final_output.token_ids,
                        logprobs=final_output.logprobs,
                        tokenizer=tokenizer,
                        top_logprobs=request.top_logprobs,
                    )
                    if request.is_include_output_logprobs()
                    else None
                ),
842
            )
843
            message_item = ResponseOutputMessage(
844
845
846
847
848
849
                id=f"msg_{random_uuid()}",
                content=[output_text],
                role="assistant",
                status="completed",
                type="message",
            )
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
        outputs = []

        if reasoning_item:
            outputs.append(reasoning_item)
        if message_item:
            outputs.append(message_item)
        if tool_calls:
            tool_call_items = [
                ResponseFunctionToolCall(
                    id=f"fc_{random_uuid()}",
                    call_id=f"call_{random_uuid()}",
                    type="function_call",
                    status="completed",
                    name=tool_call.name,
                    arguments=tool_call.arguments,
                )
                for tool_call in tool_calls
            ]
            outputs.extend(tool_call_items)
        return outputs
870
871
872
873
874

    def _make_response_output_items_with_harmony(
        self,
        context: HarmonyContext,
    ) -> list[ResponseOutputItem]:
875
        output_items: list[ResponseOutputItem] = []
876
877
878
879
880
881
882
883
884
        num_init_messages = context.num_init_messages
        for msg in context.messages[num_init_messages:]:
            output_items.extend(parse_output_message(msg))
        # Handle the generation stopped in the middle (if any).
        last_items = parse_remaining_state(context.parser)
        if last_items:
            output_items.extend(last_items)
        return output_items

885
886
887
    def _construct_input_messages(
        self,
        request: ResponsesRequest,
888
        prev_response: ResponsesResponse | None = None,
889
890
891
    ) -> list[ChatCompletionMessageParam]:
        messages: list[ChatCompletionMessageParam] = []
        if request.instructions:
892
893
894
895
896
897
            messages.append(
                {
                    "role": "system",
                    "content": request.instructions,
                }
            )
898
899
900
901
902
903
904
905
906
907
908
909

        # Prepend the conversation history.
        if prev_response is not None:
            # Add the previous messages.
            prev_msg = self.msg_store[prev_response.id]
            messages.extend(prev_msg)

            # Add the previous output.
            for output_item in prev_response.output:
                # NOTE: We skip the reasoning output.
                if isinstance(output_item, ResponseOutputMessage):
                    for content in output_item.content:
910
911
912
913
914
915
                        messages.append(
                            {
                                "role": "assistant",
                                "content": content.text,
                            }
                        )
916
917

        # Append the new input.
918
        # Responses API supports simple text inputs without chat format.
919
920
921
        if isinstance(request.input, str):
            messages.append({"role": "user", "content": request.input})
        else:
922
923
            for item in request.input:
                messages.append(construct_chat_message_with_tool_call(item))
924
925
        return messages

926
    def _construct_harmony_system_input_message(
927
        self, request: ResponsesRequest, with_custom_tools: bool, tool_types: set[str]
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
    ) -> OpenAIHarmonyMessage:
        reasoning_effort = request.reasoning.effort if request.reasoning else None
        enable_browser = (
            "web_search_preview" in tool_types
            and self.tool_server is not None
            and self.tool_server.has_tool("browser")
        )
        enable_code_interpreter = (
            "code_interpreter" in tool_types
            and self.tool_server is not None
            and self.tool_server.has_tool("python")
        )
        enable_container = (
            "container" in tool_types
            and self.tool_server is not None
            and self.tool_server.has_tool("container")
        )
        sys_msg = get_system_message(
            reasoning_effort=reasoning_effort,
            browser_description=(
                self.tool_server.get_tool_description("browser")
                if enable_browser and self.tool_server is not None
                else None
            ),
            python_description=(
                self.tool_server.get_tool_description("python")
                if enable_code_interpreter and self.tool_server is not None
                else None
            ),
            container_description=(
                self.tool_server.get_tool_description("container")
                if enable_container and self.tool_server is not None
                else None
            ),
            instructions=request.instructions,
            with_custom_tools=with_custom_tools,
        )
        return sys_msg

967
968
969
    def _construct_input_messages_with_harmony(
        self,
        request: ResponsesRequest,
970
        prev_response: ResponsesResponse | None,
971
972
973
974
    ) -> list[OpenAIHarmonyMessage]:
        messages: list[OpenAIHarmonyMessage] = []
        if prev_response is None:
            # New conversation.
975
            tool_types = extract_tool_types(request.tools)
976
            with_custom_tools = has_custom_tools(tool_types)
977
978
979

            sys_msg = self._construct_harmony_system_input_message(
                request, with_custom_tools, tool_types
980
981
            )
            messages.append(sys_msg)
982
983
            if with_custom_tools:
                dev_msg = get_developer_message(
984
985
                    instructions=request.instructions, tools=request.tools
                )
986
                messages.append(dev_msg)
987
988
            messages += construct_harmony_previous_input_messages(request)

989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
        else:
            # Continue the previous conversation.
            # FIXME(woosuk): Currently, request params like reasoning and
            # instructions are ignored.
            prev_msgs = self.msg_store[prev_response.id]
            # Remove the previous chain-of-thoughts if there is a new "final"
            # message. Note that this also removes these messages from the
            # msg_store.
            if len(prev_msgs) > 0:
                last_msg = prev_msgs[-1]
                assert isinstance(last_msg, OpenAIHarmonyMessage)
                if last_msg.channel == "final":
                    prev_final_msg_idx = -1
                    for i in range(len(prev_msgs) - 2, -1, -1):
                        prev_msg_i = prev_msgs[i]
                        assert isinstance(prev_msg_i, OpenAIHarmonyMessage)
                        if prev_msg_i.channel == "final":
                            prev_final_msg_idx = i
                            break
1008
1009
                    recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :]
                    del prev_msgs[prev_final_msg_idx + 1 :]
1010
1011
1012
1013
1014
1015
                    for msg in recent_turn_msgs:
                        assert isinstance(msg, OpenAIHarmonyMessage)
                        if msg.channel != "analysis":
                            prev_msgs.append(msg)
            messages.extend(prev_msgs)
        # Append the new input.
co63oc's avatar
co63oc committed
1016
        # Responses API supports simple text inputs without chat format.
1017
1018
1019
1020
1021
1022
1023
1024
        if isinstance(request.input, str):
            messages.append(get_user_message(request.input))
        else:
            if prev_response is not None:
                prev_outputs = copy(prev_response.output)
            else:
                prev_outputs = []
            for response_msg in request.input:
1025
                messages.append(parse_response_input(response_msg, prev_outputs))
1026
                # User passes in a tool call request and its output. We need
1027
1028
1029
1030
1031
1032
1033
                # to add the tool call request to prev_outputs so that the
                # parse_response_input can find the tool call request when
                # parsing the tool call output.
                if isinstance(response_msg, ResponseFunctionToolCall):
                    prev_outputs.append(response_msg)
        return messages

1034
1035
1036
1037
1038
1039
    async def _run_background_request_stream(
        self,
        request: ResponsesRequest,
        *args,
        **kwargs,
    ):
1040
        event_deque: deque[StreamingResponsesResponse] = deque()
1041
1042
1043
1044
        new_event_signal = asyncio.Event()
        self.event_store[request.request_id] = (event_deque, new_event_signal)
        response = None
        try:
1045
            generator = self.responses_stream_generator(request, *args, **kwargs)
1046
1047
1048
1049
            async for event in generator:
                event_deque.append(event)
                new_event_signal.set()  # Signal new event available
        except Exception as e:
1050
            logger.exception("Background request failed for %s", request.request_id)
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
            response = self.create_error_response(str(e))
        finally:
            new_event_signal.set()

        if response is not None and isinstance(response, ErrorResponse):
            # If the request has failed, update the status to "failed".
            response_id = request.request_id
            async with self.response_store_lock:
                stored_response = self.response_store.get(response_id)
                assert stored_response is not None
                if stored_response.status not in ("completed", "cancelled"):
                    stored_response.status = "failed"

1064
1065
1066
1067
1068
1069
1070
    async def _run_background_request(
        self,
        request: ResponsesRequest,
        *args,
        **kwargs,
    ):
        try:
1071
            response = await self.responses_full_generator(request, *args, **kwargs)
1072
        except Exception as e:
1073
            logger.exception("Background request failed for %s", request.request_id)
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
            response = self.create_error_response(str(e))

        if isinstance(response, ErrorResponse):
            # If the request has failed, update the status to "failed".
            response_id = request.request_id
            async with self.response_store_lock:
                stored_response = self.response_store.get(response_id)
                assert stored_response is not None
                if stored_response.status not in ("completed", "cancelled"):
                    stored_response.status = "failed"

1085
1086
1087
    async def responses_background_stream_generator(
        self,
        response_id: str,
1088
        starting_after: int | None = None,
1089
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
        if response_id not in self.event_store:
            raise ValueError(f"Unknown response_id: {response_id}")

        event_deque, new_event_signal = self.event_store[response_id]
        start_index = 0 if starting_after is None else starting_after + 1
        current_index = start_index

        while True:
            new_event_signal.clear()

            # Yield existing events from start_index
            while current_index < len(event_deque):
                event = event_deque[current_index]
                yield event
1104
                if getattr(event, "type", "unknown") == "response.completed":
1105
                    return
1106
1107
1108
1109
                current_index += 1

            await new_event_signal.wait()

1110
1111
1112
    async def retrieve_responses(
        self,
        response_id: str,
1113
1114
1115
1116
1117
1118
1119
        starting_after: int | None,
        stream: bool | None,
    ) -> (
        ErrorResponse
        | ResponsesResponse
        | AsyncGenerator[StreamingResponsesResponse, None]
    ):
1120
1121
1122
1123
1124
        async with self.response_store_lock:
            response = self.response_store.get(response_id)

        if response is None:
            return self._make_not_found_error(response_id)
1125
1126
1127
1128
1129
1130

        if stream:
            return self.responses_background_stream_generator(
                response_id,
                starting_after,
            )
1131
1132
1133
1134
1135
        return response

    async def cancel_responses(
        self,
        response_id: str,
1136
    ) -> ErrorResponse | ResponsesResponse:
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
        async with self.response_store_lock:
            response = self.response_store.get(response_id)
            if response is None:
                return self._make_not_found_error(response_id)

            prev_status = response.status
            if prev_status not in ("queued", "in_progress"):
                return self.create_error_response(
                    err_type="invalid_request_error",
                    message="Cannot cancel a synchronous response.",
                )

            # Update the status to "cancelled".
            response.status = "cancelled"

        # Abort the request.
1153
        if task := self.background_tasks.get(response_id):
1154
1155
1156
1157
            task.cancel()
            try:
                await task
            except asyncio.CancelledError:
1158
                logger.exception("Background task for %s was cancelled", response_id)
1159
1160
1161
1162
1163
1164
1165
1166
        return response

    def _make_not_found_error(self, response_id: str) -> ErrorResponse:
        return self.create_error_response(
            err_type="invalid_request_error",
            message=f"Response with id '{response_id}' not found.",
            status_code=HTTPStatus.NOT_FOUND,
        )
1167
1168
1169
1170

    def _make_store_not_supported_error(self) -> ErrorResponse:
        return self.create_error_response(
            err_type="invalid_request_error",
1171
1172
1173
1174
1175
1176
            message=(
                "`store=True` (default) is not supported. Please set "
                "`store=False` in Responses API or set "
                "`VLLM_ENABLE_RESPONSES_API_STORE=1` in the env var when "
                "starting the vLLM server."
            ),
1177
1178
            status_code=HTTPStatus.BAD_REQUEST,
        )
1179

1180
    async def _process_simple_streaming_events(
1181
1182
1183
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1184
        result_generator: AsyncIterator[ConversationContext | None],
1185
1186
1187
1188
        context: ConversationContext,
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
1189
        created_time: int,
1190
        _increment_sequence_number_and_return: Callable[
1191
1192
            [StreamingResponsesResponse], StreamingResponsesResponse
        ],
1193
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
        current_content_index = 0
        current_output_index = 0
        current_item_id = ""
        reasoning_parser = None
        if self.reasoning_parser:
            reasoning_parser = self.reasoning_parser(tokenizer)
        previous_text = ""
        previous_token_ids: list[int] = []
        first_delta_sent = False
        previous_delta_messages: list[DeltaMessage] = []
        async for ctx in result_generator:
            assert isinstance(ctx, SimpleContext)
            if ctx.last_output is None:
                continue
            if ctx.last_output.outputs:
                output = ctx.last_output.outputs[0]
                if reasoning_parser:
1211
                    delta_message = (
1212
                        reasoning_parser.extract_reasoning_content_streaming(
1213
1214
1215
1216
1217
1218
1219
                            previous_text=previous_text,
                            current_text=previous_text + output.text,
                            delta_text=output.text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=previous_token_ids + output.token_ids,
                            delta_token_ids=output.token_ids,
                        )
1220
1221
                    )
                else:
1222
1223
1224
                    delta_message = DeltaMessage(
                        content=output.text,
                    )
1225
1226
1227
1228
1229
1230
1231
                previous_text += output.text
                previous_token_ids += output.token_ids
                if not delta_message:
                    continue
                if not first_delta_sent:
                    current_item_id = str(uuid.uuid4())
                    if delta_message.reasoning_content:
1232
                        yield _increment_sequence_number_and_return(
1233
1234
1235
1236
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1237
                                item=ResponseReasoningItem(
1238
1239
1240
1241
1242
                                    type="reasoning",
                                    id=current_item_id,
                                    summary=[],
                                    status="in_progress",
                                ),
1243
1244
                            )
                        )
1245
                    else:
1246
                        yield _increment_sequence_number_and_return(
1247
1248
1249
1250
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1251
                                item=ResponseOutputMessage(
1252
1253
1254
1255
1256
1257
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[],
                                    status="in_progress",
                                ),
1258
1259
                            )
                        )
1260
                    yield _increment_sequence_number_and_return(
1261
                        ResponseContentPartAddedEvent(
1262
1263
1264
1265
1266
                            type="response.content_part.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            content_index=current_content_index,
1267
                            part=ResponseOutputText(
1268
1269
1270
1271
1272
                                type="output_text",
                                text="",
                                annotations=[],
                                logprobs=[],
                            ),
1273
1274
                        )
                    )
1275
1276
1277
1278
1279
1280
                    current_content_index += 1
                    first_delta_sent = True
                # todo(kebe7jun) tool call support

                # check delta message and previous delta message are
                # same as content or reasoning content
1281
1282
1283
1284
1285
                if (
                    previous_delta_messages
                    and previous_delta_messages[-1].reasoning_content is not None
                    and delta_message.content is not None
                ):
1286
1287
                    # from reasoning to normal content, send done
                    # event for reasoning
1288
1289
1290
1291
1292
                    reason_content = "".join(
                        pm.reasoning_content
                        for pm in previous_delta_messages
                        if pm.reasoning_content is not None
                    )
1293
                    yield _increment_sequence_number_and_return(
1294
1295
1296
1297
1298
1299
1300
                        ResponseReasoningTextDoneEvent(
                            type="response.reasoning_text.done",
                            item_id=current_item_id,
                            sequence_number=-1,
                            output_index=current_output_index,
                            content_index=current_content_index,
                            text=reason_content,
1301
1302
                        )
                    )
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
                    current_content_index = 0
                    reasoning_item = ResponseReasoningItem(
                        type="reasoning",
                        content=[
                            ResponseReasoningTextContent(
                                text=reason_content,
                                type="reasoning_text",
                            ),
                        ],
                        status="completed",
                        id=current_item_id,
                        summary=[],
                    )
1316
                    yield _increment_sequence_number_and_return(
1317
1318
1319
1320
1321
                        ResponseOutputItemDoneEvent(
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item=reasoning_item,
1322
1323
                        )
                    )
1324
                    yield _increment_sequence_number_and_return(
1325
                        ResponseOutputItemAddedEvent(
1326
1327
1328
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
1329
                            item=ResponseOutputMessage(
1330
1331
1332
1333
1334
1335
                                id=current_item_id,
                                type="message",
                                role="assistant",
                                content=[],
                                status="in_progress",
                            ),
1336
1337
                        )
                    )
1338
1339
                    current_output_index += 1
                    current_item_id = str(uuid.uuid4())
1340
                    yield _increment_sequence_number_and_return(
1341
                        ResponseContentPartAddedEvent(
1342
1343
1344
1345
1346
                            type="response.content_part.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            content_index=current_content_index,
1347
                            part=ResponseOutputText(
1348
1349
1350
1351
1352
                                type="output_text",
                                text="",
                                annotations=[],
                                logprobs=[],
                            ),
1353
1354
                        )
                    )
1355
1356
1357
                    current_content_index += 1
                    # reset previous delta messages
                    previous_delta_messages = []
1358

1359
                if delta_message.reasoning_content is not None:
1360
                    yield _increment_sequence_number_and_return(
1361
1362
1363
1364
1365
1366
1367
                        ResponseReasoningTextDeltaEvent(
                            type="response.reasoning_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=delta_message.reasoning_content,
1368
1369
                        )
                    )
1370
                elif delta_message.content is not None:
1371
                    yield _increment_sequence_number_and_return(
1372
                        ResponseTextDeltaEvent(
1373
1374
1375
1376
1377
1378
                            type="response.output_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=delta_message.content,
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
                            logprobs=(
                                self._create_stream_response_logprobs(
                                    token_ids=output.token_ids,
                                    logprobs=output.logprobs,
                                    tokenizer=tokenizer,
                                    top_logprobs=request.top_logprobs,
                                )
                                if request.is_include_output_logprobs()
                                else []
                            ),
1389
1390
                        )
                    )
1391
1392
1393
1394
1395
                current_content_index += 1

                previous_delta_messages.append(delta_message)
        if previous_delta_messages:
            if previous_delta_messages[-1].reasoning_content is not None:
1396
1397
1398
1399
1400
                reason_content = "".join(
                    pm.reasoning_content
                    for pm in previous_delta_messages
                    if pm.reasoning_content is not None
                )
1401
                yield _increment_sequence_number_and_return(
1402
1403
1404
1405
1406
1407
1408
                    ResponseReasoningTextDoneEvent(
                        type="response.reasoning_text.done",
                        item_id=current_item_id,
                        sequence_number=-1,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        text=reason_content,
1409
1410
                    )
                )
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
                current_content_index += 1
                reasoning_item = ResponseReasoningItem(
                    type="reasoning",
                    content=[
                        ResponseReasoningTextContent(
                            text=reason_content,
                            type="reasoning_text",
                        ),
                    ],
                    status="completed",
                    id=current_item_id,
                    summary=[],
                )
1424
                yield _increment_sequence_number_and_return(
1425
1426
1427
1428
1429
                    ResponseOutputItemDoneEvent(
                        type="response.output_item.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        item=reasoning_item,
1430
1431
                    )
                )
1432
            elif previous_delta_messages[-1].content is not None:
1433
1434
1435
1436
1437
                final_content = "".join(
                    pm.content
                    for pm in previous_delta_messages
                    if pm.content is not None
                )
1438
                yield _increment_sequence_number_and_return(
1439
                    ResponseTextDoneEvent(
1440
1441
1442
1443
1444
1445
1446
                        type="response.output_text.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        text=final_content,
                        logprobs=[],
                        item_id=current_item_id,
1447
1448
                    )
                )
1449
1450
1451
1452
1453
1454
                current_content_index += 1
                part = ResponseOutputText(
                    text=final_content,
                    type="output_text",
                    annotations=[],
                )
1455
                yield _increment_sequence_number_and_return(
1456
                    ResponseContentPartDoneEvent(
1457
1458
1459
1460
1461
1462
                        type="response.content_part.done",
                        sequence_number=-1,
                        item_id=current_item_id,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        part=part,
1463
1464
                    )
                )
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
                current_content_index += 1
                item = ResponseOutputMessage(
                    type="message",
                    role="assistant",
                    content=[
                        part,
                    ],
                    status="completed",
                    id=current_item_id,
                    summary=[],
                )
1476
                yield _increment_sequence_number_and_return(
1477
1478
1479
1480
1481
                    ResponseOutputItemDoneEvent(
                        type="response.output_item.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        item=item,
1482
1483
                    )
                )
1484
1485
1486
1487
1488

    async def _process_harmony_streaming_events(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1489
        result_generator: AsyncIterator[ConversationContext | None],
1490
1491
1492
1493
1494
        context: ConversationContext,
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
        created_time: int,
1495
        _increment_sequence_number_and_return: Callable[
1496
1497
            [StreamingResponsesResponse], StreamingResponsesResponse
        ],
1498
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1499
        current_content_index = -1
1500
        current_output_index = 0
1501
        current_item_id: str = ""
1502
        sent_output_item_added = False
1503
        is_first_function_call_delta = False
1504
1505
1506
1507
1508
1509
        async for ctx in result_generator:
            assert isinstance(ctx, StreamingHarmonyContext)

            if ctx.is_expecting_start():
                current_output_index += 1
                sent_output_item_added = False
1510
                is_first_function_call_delta = False
1511
1512
1513
                if len(ctx.parser.messages) > 0:
                    previous_item = ctx.parser.messages[-1]
                    if previous_item.recipient is not None:
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
                        # Deal with tool call
                        if previous_item.recipient.startswith("functions."):
                            function_name = previous_item.recipient[len("functions.") :]
                            yield _increment_sequence_number_and_return(
                                ResponseFunctionCallArgumentsDoneEvent(
                                    type="response.function_call_arguments.done",
                                    arguments=previous_item.content[0].text,
                                    name=function_name,
                                    item_id=current_item_id,
                                    output_index=current_output_index,
                                    sequence_number=-1,
                                )
                            )
                            function_call_item = ResponseFunctionToolCall(
                                type="function_call",
                                arguments=previous_item.content[0].text,
                                name=function_name,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                sequence_number=-1,
                                call_id=f"fc_{random_uuid()}",
                                status="completed",
                            )
                            yield _increment_sequence_number_and_return(
                                ResponseOutputItemDoneEvent(
                                    type="response.output_item.done",
                                    sequence_number=-1,
                                    output_index=current_output_index,
                                    item=function_call_item,
                                )
                            )
1545
                    elif previous_item.channel == "analysis":
1546
1547
1548
1549
                        content = ResponseReasoningTextContent(
                            text=previous_item.content[0].text,
                            type="reasoning_text",
                        )
1550
1551
                        reasoning_item = ResponseReasoningItem(
                            type="reasoning",
1552
                            content=[content],
1553
                            status="completed",
1554
1555
                            id=current_item_id,
                            summary=[],
1556
                        )
1557
                        yield _increment_sequence_number_and_return(
1558
1559
1560
1561
1562
1563
1564
                            ResponseReasoningTextDoneEvent(
                                type="response.reasoning_text.done",
                                item_id=current_item_id,
                                sequence_number=-1,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                text=previous_item.content[0].text,
1565
1566
                            )
                        )
1567
1568
1569
1570
1571
1572
1573
1574
                        yield _increment_sequence_number_and_return(
                            ResponseReasoningPartDoneEvent(
                                type="response.reasoning_part.done",
                                sequence_number=-1,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                part=content,
1575
1576
                            )
                        )
1577
                        yield _increment_sequence_number_and_return(
1578
1579
1580
1581
1582
                            ResponseOutputItemDoneEvent(
                                type="response.output_item.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item=reasoning_item,
1583
1584
                            )
                        )
1585
1586
1587
1588
1589
1590
                    elif previous_item.channel == "final":
                        text_content = ResponseOutputText(
                            type="output_text",
                            text=previous_item.content[0].text,
                            annotations=[],
                        )
1591
                        yield _increment_sequence_number_and_return(
1592
                            ResponseTextDoneEvent(
1593
1594
1595
1596
1597
1598
1599
                                type="response.output_text.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                text=previous_item.content[0].text,
                                logprobs=[],
                                item_id=current_item_id,
1600
1601
                            )
                        )
1602
                        yield _increment_sequence_number_and_return(
1603
1604
1605
1606
1607
1608
1609
                            ResponseContentPartDoneEvent(
                                type="response.content_part.done",
                                sequence_number=-1,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                part=text_content,
1610
1611
                            )
                        )
1612
                        yield _increment_sequence_number_and_return(
1613
                            ResponseOutputItemDoneEvent(
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
                                type="response.output_item.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item=ResponseOutputMessage(
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[text_content],
                                    status="completed",
                                ),
1624
1625
                            )
                        )
1626

1627
            # stream the output of a harmony message
1628
            if ctx.parser.last_content_delta:
1629
1630
1631
1632
                if (
                    ctx.parser.current_channel == "final"
                    and ctx.parser.current_recipient is None
                ):
1633
1634
                    if not sent_output_item_added:
                        sent_output_item_added = True
1635
                        current_item_id = f"msg_{random_uuid()}"
1636
                        yield _increment_sequence_number_and_return(
1637
1638
1639
1640
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1641
                                item=ResponseOutputMessage(
1642
1643
1644
1645
1646
1647
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[],
                                    status="in_progress",
                                ),
1648
1649
                            )
                        )
1650
                        current_content_index += 1
1651
                        yield _increment_sequence_number_and_return(
1652
1653
1654
1655
1656
1657
                            ResponseContentPartAddedEvent(
                                type="response.content_part.added",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
                                content_index=current_content_index,
1658
                                part=ResponseOutputText(
1659
1660
1661
1662
1663
                                    type="output_text",
                                    text="",
                                    annotations=[],
                                    logprobs=[],
                                ),
1664
1665
                            )
                        )
1666
                    yield _increment_sequence_number_and_return(
1667
                        ResponseTextDeltaEvent(
1668
1669
1670
1671
1672
1673
1674
1675
                            type="response.output_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
                            # TODO, use logprobs from ctx.last_request_output
                            logprobs=[],
1676
1677
1678
1679
1680
1681
                        )
                    )
                elif (
                    ctx.parser.current_channel == "analysis"
                    and ctx.parser.current_recipient is None
                ):
1682
1683
                    if not sent_output_item_added:
                        sent_output_item_added = True
1684
                        current_item_id = f"msg_{random_uuid()}"
1685
                        yield _increment_sequence_number_and_return(
1686
1687
1688
1689
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1690
                                item=ResponseReasoningItem(
1691
1692
1693
1694
1695
                                    type="reasoning",
                                    id=current_item_id,
                                    summary=[],
                                    status="in_progress",
                                ),
1696
1697
                            )
                        )
1698
                        current_content_index += 1
1699
                        yield _increment_sequence_number_and_return(
1700
1701
                            ResponseReasoningPartAddedEvent(
                                type="response.reasoning_part.added",
1702
1703
1704
1705
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
                                content_index=current_content_index,
1706
                                part=ResponseReasoningTextContent(
1707
                                    text="",
1708
                                    type="reasoning_text",
1709
                                ),
1710
1711
                            )
                        )
1712
                    yield _increment_sequence_number_and_return(
1713
1714
1715
1716
1717
1718
1719
                        ResponseReasoningTextDeltaEvent(
                            type="response.reasoning_text.delta",
                            item_id=current_item_id,
                            output_index=current_output_index,
                            content_index=current_content_index,
                            delta=ctx.parser.last_content_delta,
                            sequence_number=-1,
1720
1721
                        )
                    )
1722
1723
1724
                # built-in tools will be triggered on the analysis channel
                # However, occasionally built-in tools will
                # still be output to commentary.
1725
1726
1727
1728
                elif (
                    ctx.parser.current_channel == "commentary"
                    or ctx.parser.current_channel == "analysis"
                ) and ctx.parser.current_recipient == "python":
1729
1730
                    if not sent_output_item_added:
                        sent_output_item_added = True
1731
                        current_item_id = f"tool_{random_uuid()}"
1732
                        yield _increment_sequence_number_and_return(
1733
1734
1735
1736
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1737
                                item=ResponseCodeInterpreterToolCallParam(
1738
1739
1740
1741
1742
1743
1744
                                    type="code_interpreter_call",
                                    id=current_item_id,
                                    code=None,
                                    container_id="auto",
                                    outputs=None,
                                    status="in_progress",
                                ),
1745
1746
                            )
                        )
1747
                        yield _increment_sequence_number_and_return(
1748
                            ResponseCodeInterpreterCallInProgressEvent(
1749
                                type="response.code_interpreter_call.in_progress",
1750
1751
1752
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
1753
1754
                            )
                        )
1755
                    yield _increment_sequence_number_and_return(
1756
1757
1758
1759
1760
1761
                        ResponseCodeInterpreterCallCodeDeltaEvent(
                            type="response.code_interpreter_call_code.delta",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
1762
1763
                        )
                    )
1764
1765

            # stream tool call outputs
1766
1767
            if ctx.is_assistant_action_turn() and len(ctx.parser.messages) > 0:
                previous_item = ctx.parser.messages[-1]
1768
1769
1770
1771
1772
1773
1774
                if (
                    self.tool_server is not None
                    and self.tool_server.has_tool("browser")
                    and previous_item.recipient is not None
                    and previous_item.recipient.startswith("browser.")
                ):
                    function_name = previous_item.recipient[len("browser.") :]
1775
1776
1777
                    action = None
                    parsed_args = json.loads(previous_item.content[0].text)
                    if function_name == "search":
1778
                        action = response_function_web_search.ActionSearch(
1779
1780
                            type="search",
                            query=parsed_args["query"],
1781
                        )
1782
                    elif function_name == "open":
1783
1784
1785
1786
1787
                        action = response_function_web_search.ActionOpenPage(
                            type="open_page",
                            # TODO: translate to url
                            url=f"cursor:{parsed_args.get('cursor', '')}",
                        )
1788
                    elif function_name == "find":
1789
1790
1791
1792
1793
1794
                        action = response_function_web_search.ActionFind(
                            type="find",
                            pattern=parsed_args["pattern"],
                            # TODO: translate to url
                            url=f"cursor:{parsed_args.get('cursor', '')}",
                        )
1795
                    else:
1796
                        raise ValueError(f"Unknown function name: {function_name}")
1797

1798
                    current_item_id = f"tool_{random_uuid()}"
1799
                    yield _increment_sequence_number_and_return(
1800
                        ResponseOutputItemAddedEvent(
1801
1802
1803
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
1804
                            item=response_function_web_search.ResponseFunctionWebSearch(
1805
1806
1807
1808
1809
1810
                                # TODO: generate a unique id for web search call
                                type="web_search_call",
                                id=current_item_id,
                                action=action,
                                status="in_progress",
                            ),
1811
1812
                        )
                    )
1813
                    yield _increment_sequence_number_and_return(
1814
1815
1816
1817
1818
                        ResponseWebSearchCallInProgressEvent(
                            type="response.web_search_call.in_progress",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1819
1820
                        )
                    )
1821
                    yield _increment_sequence_number_and_return(
1822
1823
1824
1825
1826
                        ResponseWebSearchCallSearchingEvent(
                            type="response.web_search_call.searching",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1827
1828
                        )
                    )
1829
1830

                    # enqueue
1831
                    yield _increment_sequence_number_and_return(
1832
1833
1834
1835
1836
                        ResponseWebSearchCallCompletedEvent(
                            type="response.web_search_call.completed",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1837
1838
                        )
                    )
1839
                    yield _increment_sequence_number_and_return(
1840
                        ResponseOutputItemDoneEvent(
1841
1842
1843
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
1844
                            item=ResponseFunctionWebSearch(
1845
1846
1847
1848
1849
                                type="web_search_call",
                                id=current_item_id,
                                action=action,
                                status="completed",
                            ),
1850
1851
                        )
                    )
1852

1853
1854
1855
1856
1857
1858
                if (
                    self.tool_server is not None
                    and self.tool_server.has_tool("python")
                    and previous_item.recipient is not None
                    and previous_item.recipient.startswith("python")
                ):
1859
                    yield _increment_sequence_number_and_return(
1860
1861
1862
1863
1864
                        ResponseCodeInterpreterCallCodeDoneEvent(
                            type="response.code_interpreter_call_code.done",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1865
                            code=previous_item.content[0].text,
1866
1867
                        )
                    )
1868
                    yield _increment_sequence_number_and_return(
1869
1870
1871
1872
1873
                        ResponseCodeInterpreterCallInterpretingEvent(
                            type="response.code_interpreter_call.interpreting",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1874
1875
                        )
                    )
1876
                    yield _increment_sequence_number_and_return(
1877
1878
1879
1880
1881
                        ResponseCodeInterpreterCallCompletedEvent(
                            type="response.code_interpreter_call.completed",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1882
1883
                        )
                    )
1884
                    yield _increment_sequence_number_and_return(
1885
                        ResponseOutputItemDoneEvent(
1886
1887
1888
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
1889
                            item=ResponseCodeInterpreterToolCallParam(
1890
1891
1892
1893
1894
1895
1896
1897
                                type="code_interpreter_call",
                                id=current_item_id,
                                code=previous_item.content[0].text,
                                container_id="auto",
                                # TODO: add outputs here
                                outputs=[],
                                status="completed",
                            ),
1898
1899
                        )
                    )
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
            # developer tools will be triggered on the commentary channel
            # and recipient starts with "functions.TOOL_NAME"
            if (
                ctx.parser.current_channel == "commentary"
                and ctx.parser.current_recipient
                and ctx.parser.current_recipient.startswith("functions.")
            ):
                if is_first_function_call_delta is False:
                    is_first_function_call_delta = True
                    fc_name = ctx.parser.current_recipient[len("functions.") :]
                    tool_call_item = ResponseFunctionToolCall(
                        name=fc_name,
                        type="function_call",
                        id=current_item_id,
                        call_id=f"call_{random_uuid()}",
                        arguments="",
                        status="in_progress",
                    )
                    current_item_id = f"fc_{random_uuid()}"
                    yield _increment_sequence_number_and_return(
                        ResponseOutputItemAddedEvent(
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item=tool_call_item,
                        )
                    )
                else:
                    yield _increment_sequence_number_and_return(
                        ResponseFunctionCallArgumentsDeltaEvent(
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
                            output_index=current_output_index,
                            sequence_number=-1,
                            type="response.function_call_arguments.delta",
                        )
                    )
1937

1938
1939
1940
1941
    async def responses_stream_generator(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1942
        result_generator: AsyncIterator[ConversationContext | None],
1943
1944
1945
1946
        context: ConversationContext,
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
1947
        created_time: int | None = None,
1948
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1949
1950
1951
1952
1953
        # TODO:
        # 1. Handle disconnect

        created_time = created_time or int(time.time())

1954
1955
        sequence_number = 0

1956
        def _increment_sequence_number_and_return(
1957
            event: StreamingResponsesResponse,
1958
        ) -> StreamingResponsesResponse:
1959
1960
            nonlocal sequence_number
            # Set sequence_number if the event has this attribute
1961
            if hasattr(event, "sequence_number"):
1962
1963
                event.sequence_number = sequence_number
            sequence_number += 1
1964
            return event
1965

1966
        async with AsyncExitStack() as exit_stack:
1967
1968
            processer = None
            if self.use_harmony:
1969
1970
                # TODO: in streaming, we noticed this bug:
                # https://github.com/vllm-project/vllm/issues/25697
1971
                await self._initialize_tool_sessions(request, context, exit_stack)
1972
1973
1974
                processer = self._process_harmony_streaming_events
            else:
                processer = self._process_simple_streaming_events
1975
            # TODO Hanchen make sampling params to include the structural tag
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985

            initial_response = ResponsesResponse.from_request(
                request,
                sampling_params,
                model_name=model_name,
                created_time=created_time,
                output=[],
                status="in_progress",
                usage=None,
            ).model_dump()
1986
            yield _increment_sequence_number_and_return(
1987
1988
1989
1990
                ResponseCreatedEvent(
                    type="response.created",
                    sequence_number=-1,
                    response=initial_response,
1991
1992
                )
            )
1993
            yield _increment_sequence_number_and_return(
1994
1995
1996
1997
                ResponseInProgressEvent(
                    type="response.in_progress",
                    sequence_number=-1,
                    response=initial_response,
1998
1999
                )
            )
2000

2001
            async for event_data in processer(
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
                created_time,
                _increment_sequence_number_and_return,
            ):
2012
                yield event_data
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029

            async def empty_async_generator():
                # A hack to trick Python to think this is a generator but
                # in fact it immediately returns.
                if False:
                    yield

            final_response = await self.responses_full_generator(
                request,
                sampling_params,
                empty_async_generator(),
                context,
                model_name,
                tokenizer,
                request_metadata,
                created_time=created_time,
            )
2030
            yield _increment_sequence_number_and_return(
2031
                ResponseCompletedEvent(
2032
2033
                    type="response.completed",
                    sequence_number=-1,
2034
                    response=final_response,
2035
2036
                )
            )