serving_responses.py 81.3 KB
Newer Older
1
2
3
4
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import asyncio
5
import json
6
import time
7
import uuid
8
from collections import deque
9
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
10
from contextlib import AsyncExitStack
11
from copy import copy
12
from http import HTTPStatus
13
from typing import Final
14
15
16

import jinja2
from fastapi import Request
17
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
    ResponseCodeInterpreterCallInterpretingEvent,
23
24
25
    ResponseCodeInterpreterToolCallParam,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
26
27
    ResponseFunctionCallArgumentsDeltaEvent,
    ResponseFunctionCallArgumentsDoneEvent,
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    ResponseFunctionToolCall,
    ResponseFunctionWebSearch,
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponseOutputMessage,
    ResponseOutputText,
    ResponseReasoningItem,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseTextDeltaEvent,
    ResponseTextDoneEvent,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
    response_function_web_search,
    response_text_delta_event,
)
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
48
from openai.types.responses.response_reasoning_item import (
49
50
    Content as ResponseReasoningTextContent,
)
51
from openai_harmony import Message as OpenAIHarmonyMessage
52

53
from vllm import envs
54
from vllm.engine.protocol import EngineClient
55
56
57
58
59
60
61
62
63
64
from vllm.entrypoints.chat_utils import (
    ChatCompletionMessageParam,
    ChatTemplateContentFormatOption,
)
from vllm.entrypoints.context import (
    ConversationContext,
    HarmonyContext,
    SimpleContext,
    StreamingHarmonyContext,
)
65
from vllm.entrypoints.harmony_utils import (
66
67
68
69
70
71
72
73
74
75
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_system_message,
    get_user_message,
    has_custom_tools,
    parse_output_message,
    parse_remaining_state,
    parse_response_input,
    render_for_completion,
)
76
from vllm.entrypoints.logger import RequestLogger
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
from vllm.entrypoints.openai.protocol import (
    DeltaMessage,
    ErrorResponse,
    InputTokensDetails,
    OutputTokensDetails,
    RequestResponseMetadata,
    ResponseCompletedEvent,
    ResponseCreatedEvent,
    ResponseInProgressEvent,
    ResponseReasoningPartAddedEvent,
    ResponseReasoningPartDoneEvent,
    ResponsesRequest,
    ResponsesResponse,
    ResponseUsage,
    StreamingResponsesResponse,
)
93
94
from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
95
from vllm.entrypoints.tool_server import ToolServer
96
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
97
from vllm.logger import init_logger
98
99
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
100
from vllm.outputs import CompletionOutput
101
102
103
104
105
106
107
108
109
110
111
112
113
from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.utils import random_uuid

logger = init_logger(__name__)


class OpenAIServingResponses(OpenAIServing):
    def __init__(
        self,
        engine_client: EngineClient,
        models: OpenAIServingModels,
        *,
114
115
        request_logger: RequestLogger | None,
        chat_template: str | None,
116
117
118
119
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
        reasoning_parser: str = "",
        enable_auto_tools: bool = False,
120
121
        tool_parser: str | None = None,
        tool_server: ToolServer | None = None,
122
123
        enable_prompt_tokens_details: bool = False,
        enable_force_include_usage: bool = False,
124
        enable_log_outputs: bool = False,
125
        log_error_stack: bool = False,
126
127
128
129
130
131
    ) -> None:
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
132
            log_error_stack=log_error_stack,
133
134
135
136
        )

        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
137
        self.enable_log_outputs = enable_log_outputs
138

139
140
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
141
        )
142
143
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
        self.enable_force_include_usage = enable_force_include_usage
144
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
145
146
147
        if self.default_sampling_params:
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
148
149
150
151
152
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
153

154
155
156
157
158
        # If False (default), the "store" option is (silently) ignored and the
        # response is not stored. If True, the response is stored in memory.
        # NOTE(woosuk): This may not be intuitive for users, as the default
        # behavior in OpenAI's Responses API is to store the response, but
        # vLLM's default behavior is not.
159
        self.enable_store = envs.VLLM_ENABLE_RESPONSES_API_STORE
160
161
162
163
        if self.enable_store:
            logger.warning_once(
                "`VLLM_ENABLE_RESPONSES_API_STORE` is enabled. This may "
                "cause a memory leak since we never remove responses from "
164
165
                "the store."
            )
166

167
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
168
        if self.use_harmony:
169
170
171
172
            logger.warning(
                "For gpt-oss, we ignore --enable-auto-tool-choice "
                "and always enable tool use."
            )
173
174
175
176
177
            # OpenAI models have two EOS-like tokens: <|return|> and <|call|>.
            # We need to add them to the stop token ids.
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
178
179
                get_stop_tokens_for_assistant_actions()
            )
180
181
182
183
184

        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
185
                '"auto" tool choice has been enabled please note that while'
186
                " the parallel_tool_calls client option is preset for "
187
188
                "compatibility reasons, it will be ignored."
            )
189

190
        # HACK(woosuk): This is a hack. We should use a better store.
191
192
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove responses from the store.
193
194
195
196
        self.response_store: dict[str, ResponsesResponse] = {}
        self.response_store_lock = asyncio.Lock()

        # HACK(woosuk): This is a hack. We should use a better store.
197
198
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove messages from the store.
199
200
        self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}

201
202
203
        # HACK(wuhang): This is a hack. We should use a better store.
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove events from the store.
204
205
206
        self.event_store: dict[
            str, tuple[deque[StreamingResponsesResponse], asyncio.Event]
        ] = {}
207

208
209
        self.background_tasks: dict[str, asyncio.Task] = {}

210
211
        self.tool_server = tool_server

212
    def _validate_generator_input(
213
        self, engine_prompt: EngineTokensPrompt
214
    ) -> ErrorResponse | None:
215
216
217
218
219
220
        """Add validations to the input to the generator here."""
        if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
            error_message = (
                "The engine prompt length"
                f" {len(engine_prompt['prompt_token_ids'])} "
                f"exceeds the max_model_len {self.max_model_len}. "
221
222
                "Please reduce prompt."
            )
223
224
225
226
227
228
229
            return self.create_error_response(
                err_type="invalid_request_error",
                message=error_message,
                status_code=HTTPStatus.BAD_REQUEST,
            )
        return None

230
231
232
    async def create_responses(
        self,
        request: ResponsesRequest,
233
234
235
236
237
238
        raw_request: Request | None = None,
    ) -> (
        AsyncGenerator[StreamingResponsesResponse, None]
        | ResponsesResponse
        | ErrorResponse
    ):
239
240
241
242
243
244
245
246
247
248
249
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
            logger.error("Error with model %s", error_check_ret)
            return error_check_ret

        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

250
        if request.store and not self.enable_store:
251
252
253
254
255
256
257
258
            if request.background:
                return self.create_error_response(
                    err_type="invalid_request_error",
                    message=(
                        "This vLLM engine does not support `store=True` and "
                        "therefore does not support the background mode. To "
                        "enable these features, set the environment variable "
                        "`VLLM_ENABLE_RESPONSES_API_STORE=1` when launching "
259
260
                        "the vLLM server."
                    ),
261
262
263
264
265
266
267
268
269
                    status_code=HTTPStatus.BAD_REQUEST,
                )
            # Disable the store option.
            # NOTE(woosuk): Although returning an error is possible, we opted
            # to implicitly disable store and process the request anyway, as
            # we assume most users do not intend to actually store the response
            # (i.e., their request's `store=True` just because it's the default
            # value).
            request.store = False
270
271
272
273
274
275
        if self.use_harmony and request.is_include_output_logprobs():
            return self.create_error_response(
                err_type="invalid_request_error",
                message="logprobs are not supported with gpt-oss models",
                status_code=HTTPStatus.BAD_REQUEST,
            )
276

277
278
279
280
281
282
283
284
285
286
287
        # Handle the previous response ID.
        prev_response_id = request.previous_response_id
        if prev_response_id is not None:
            async with self.response_store_lock:
                prev_response = self.response_store.get(prev_response_id)
            if prev_response is None:
                return self._make_not_found_error(prev_response_id)
        else:
            prev_response = None

        try:
288
            lora_request = self._maybe_get_adapters(request)
289
            model_name = self.models.model_name(lora_request)
290
            tokenizer = await self.engine_client.get_tokenizer()
291

292
293
            if self.use_harmony:
                messages, request_prompts, engine_prompts = (
294
295
                    self._make_request_with_harmony(request, prev_response)
                )
296
            else:
297
298
299
                messages, request_prompts, engine_prompts = await self._make_request(
                    request, prev_response, tokenizer
                )
300

301
302
303
304
305
306
307
        except (
            ValueError,
            TypeError,
            RuntimeError,
            jinja2.TemplateError,
            NotImplementedError,
        ) as e:
308
309
310
            logger.exception("Error in preprocessing prompt inputs")
            return self.create_error_response(f"{e} {e.__cause__}")

311
        request_metadata = RequestResponseMetadata(request_id=request.request_id)
312
313
314
315
        if raw_request:
            raw_request.state.request_metadata = request_metadata

        # Schedule the request and get the result generator.
316
        generators: list[AsyncGenerator[ConversationContext, None]] = []
317
318
319
320
321
322
323

        builtin_tool_list: list[str] = []
        if self.use_harmony and self.tool_server is not None:
            if self.tool_server.has_tool("browser"):
                builtin_tool_list.append("browser")
            if self.tool_server.has_tool("python"):
                builtin_tool_list.append("python")
324
325
            if self.tool_server.has_tool("container"):
                builtin_tool_list.append("container")
326

327
328
329
330
331
332
333
        if self.tool_server is not None:
            available_tools = builtin_tool_list
        else:
            assert len(builtin_tool_list) == 0
            available_tools = []
        try:
            for i, engine_prompt in enumerate(engine_prompts):
334
335
336
337
                maybe_error = self._validate_generator_input(engine_prompt)
                if maybe_error is not None:
                    return maybe_error

338
                default_max_tokens = self.max_model_len - len(
339
340
                    engine_prompt["prompt_token_ids"]
                )
341

342
                sampling_params = request.to_sampling_params(
343
344
                    default_max_tokens, self.default_sampling_params
                )
345

346
347
348
349
350
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
351
352
353
354

                context: ConversationContext
                if self.use_harmony:
                    if request.stream:
355
                        context = StreamingHarmonyContext(messages, available_tools)
356
357
358
359
360
361
362
363
364
365
366
367
368
                    else:
                        context = HarmonyContext(messages, available_tools)
                else:
                    context = SimpleContext()
                generator = self._generate_with_builtin_tools(
                    request_id=request.request_id,
                    request_prompt=request_prompts[i],
                    engine_prompt=engine_prompt,
                    sampling_params=sampling_params,
                    context=context,
                    lora_request=lora_request,
                    priority=request.priority,
                    trace_headers=trace_headers,
369
                )
370
371
372
373
                generators.append(generator)
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
374

375
        assert len(generators) == 1
376
        (result_generator,) = generators
377
378
379
380

        # Store the input messages.
        if request.store:
            self.msg_store[request.request_id] = messages
381

382
383
384
385
386
387
388
389
390
391
392
393
394
        if request.background:
            created_time = int(time.time())
            response = ResponsesResponse.from_request(
                request,
                sampling_params,
                model_name=model_name,
                created_time=created_time,
                output=[],
                status="queued",
                usage=None,
            )
            async with self.response_store_lock:
                self.response_store[response.id] = response
395

396
            # Run the request in the background.
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
            if request.stream:
                task = asyncio.create_task(
                    self._run_background_request_stream(
                        request,
                        sampling_params,
                        result_generator,
                        context,
                        model_name,
                        tokenizer,
                        request_metadata,
                        created_time,
                    ),
                    name=f"create_{request.request_id}",
                )
            else:
                task = asyncio.create_task(
                    self._run_background_request(
                        request,
                        sampling_params,
                        result_generator,
                        context,
                        model_name,
                        tokenizer,
                        request_metadata,
                        created_time,
                    ),
                    name=f"create_{response.id}",
                )
425

426
427
428
429
            # For cleanup.
            response_id = response.id
            self.background_tasks[response_id] = task
            task.add_done_callback(
430
431
                lambda _: self.background_tasks.pop(response_id, None)
            )
432
433

            if request.stream:
434
                return self.responses_background_stream_generator(request.request_id)
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
            return response

        if request.stream:
            return self.responses_stream_generator(
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
            )

        try:
            return await self.responses_full_generator(
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
            )
        except Exception as e:
            return self.create_error_response(str(e))
460

461
462
463
    async def _make_request(
        self,
        request: ResponsesRequest,
464
        prev_response: ResponsesResponse | None,
465
466
        tokenizer: AnyTokenizer,
    ):
467
468
        if len(request.tools) > 0:
            raise NotImplementedError(
469
470
                "Tool use is not supported in Responses API without Harmony"
            )
471
472
473
474
475
476
477
478
479
480
481
482
483
484
        # Construct the input messages.
        messages = self._construct_input_messages(request, prev_response)
        _, request_prompts, engine_prompts = await self._preprocess_chat(
            request,
            tokenizer,
            messages,
            chat_template=self.chat_template,
            chat_template_content_format=self.chat_template_content_format,
        )
        return messages, request_prompts, engine_prompts

    def _make_request_with_harmony(
        self,
        request: ResponsesRequest,
485
        prev_response: ResponsesResponse | None,
486
487
488
    ):
        if request.tool_choice != "auto":
            raise NotImplementedError(
489
490
491
                "Only 'auto' tool_choice is supported in response API with Harmony"
            )
        messages = self._construct_input_messages_with_harmony(request, prev_response)
492
493
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
494
495
496
497
498

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

499
500
        return messages, [prompt_token_ids], [engine_prompt]

501
502
503
504
505
506
    async def _initialize_tool_sessions(
        self,
        request: ResponsesRequest,
        context: ConversationContext,
        exit_stack: AsyncExitStack,
    ):
507
508
509
510
        # we should only initialize the tool session if the request needs tools
        if len(request.tools) == 0:
            return
        mcp_tools = {
511
            tool.server_label: tool for tool in request.tools if tool.type == "mcp"
512
        }
513
514
515
        await context.init_tool_sessions(
            self.tool_server, exit_stack, request.request_id, mcp_tools
        )
516

517
518
519
520
    async def responses_full_generator(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
521
        result_generator: AsyncIterator[ConversationContext],
522
        context: ConversationContext,
523
524
525
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
526
527
        created_time: int | None = None,
    ) -> ErrorResponse | ResponsesResponse:
528
529
530
        if created_time is None:
            created_time = int(time.time())

531
532
        async with AsyncExitStack() as exit_stack:
            try:
533
                await self._initialize_tool_sessions(request, context, exit_stack)
534
535
536
537
538
539
540
                async for _ in result_generator:
                    pass
            except asyncio.CancelledError:
                return self.create_error_response("Client disconnected")
            except ValueError as e:
                # TODO: Use a vllm-specific Validation Error
                return self.create_error_response(str(e))
541

542
543
544
545
546
        # NOTE: Implementation of stauts is still WIP, but for now
        # we guarantee that if the status is not "completed", it is accurate.
        # "completed" is implemented as the "catch-all" for now.
        status: ResponseStatus = "completed"

547
548
        input_messages = None
        output_messages = None
549
550
551
        if self.use_harmony:
            assert isinstance(context, HarmonyContext)
            output = self._make_response_output_items_with_harmony(context)
552
            if request.enable_response_messages:
553
554
                input_messages = context.messages[: context.num_init_messages]
                output_messages = context.messages[context.num_init_messages :]
555
            num_tool_output_tokens = context.num_tool_output_tokens
556
557
558
559
560
561
562
            if len(output) > 0:
                if context.finish_reason == "length":
                    status = "incomplete"
                elif context.finish_reason == "abort":
                    status = "cancelled"
            else:
                status = "incomplete"
563
        else:
564
565
566
567
568
569
            assert isinstance(context, SimpleContext)
            final_res = context.last_output
            assert final_res is not None
            assert len(final_res.outputs) == 1
            final_output = final_res.outputs[0]

570
            output = self._make_response_output_items(request, final_output, tokenizer)
571

572
573
574
575
            # TODO: context for non-gptoss models doesn't use messages
            # so we can't get them out yet
            if request.enable_response_messages:
                raise NotImplementedError(
576
577
                    "enable_response_messages is currently only supported for gpt-oss"
                )
578
579
            # Calculate usage.
            assert final_res.prompt_token_ids is not None
580
581
            num_tool_output_tokens = 0

582
583
584
585
586
        assert isinstance(context, (SimpleContext, HarmonyContext))
        num_prompt_tokens = context.num_prompt_tokens
        num_generated_tokens = context.num_output_tokens
        num_cached_tokens = context.num_cached_tokens
        num_reasoning_tokens = context.num_reasoning_tokens
587
588
589
590

        usage = ResponseUsage(
            input_tokens=num_prompt_tokens,
            output_tokens=num_generated_tokens,
591
            total_tokens=num_prompt_tokens + num_generated_tokens,
592
            input_tokens_details=InputTokensDetails(cached_tokens=num_cached_tokens),
593
            output_tokens_details=OutputTokensDetails(
594
                reasoning_tokens=num_reasoning_tokens,
595
596
                tool_output_tokens=num_tool_output_tokens,
            ),
597
598
599
600
        )
        response = ResponsesResponse.from_request(
            request,
            sampling_params,
601
602
            input_messages=input_messages,
            output_messages=output_messages,
603
604
605
            model_name=model_name,
            created_time=created_time,
            output=output,
606
            status=status,
607
608
609
610
611
612
613
            usage=usage,
        )

        if request.store:
            async with self.response_store_lock:
                stored_response = self.response_store.get(response.id)
                # If the response is already cancelled, don't update it.
614
                if stored_response is None or stored_response.status != "cancelled":
615
616
617
                    self.response_store[response.id] = response
        return response

618
619
620
621
622
623
    def _topk_logprobs(
        self,
        logprobs: dict[int, SampleLogprob],
        top_logprobs: int,
        tokenizer: AnyTokenizer,
    ) -> list[LogprobTopLogprob]:
624
625
626
627
628
        """Returns the top-k logprobs from the logprobs dictionary."""
        out = []
        for i, (token_id, _logprob) in enumerate(logprobs.items()):
            if i >= top_logprobs:
                break
629
630
631
632
633
            text = (
                _logprob.decoded_token
                if _logprob.decoded_token is not None
                else tokenizer.decode([token_id])
            )
634
635
636
637
638
            out.append(
                LogprobTopLogprob(
                    token=text,
                    logprob=max(_logprob.logprob, -9999.0),
                    bytes=list(text.encode("utf-8", errors="replace")),
639
640
                )
            )
641
642
643
        return out

    def _create_response_logprobs(
644
645
        self,
        token_ids: Sequence[int],
646
        logprobs: SampleLogprobs | None,
647
        tokenizer: AnyTokenizer,
648
        top_logprobs: int | None = None,
649
    ) -> list[Logprob]:
650
651
        assert logprobs is not None, "logprobs must be provided"
        assert len(token_ids) == len(logprobs), (
652
653
            "token_ids and logprobs.token_ids must have the same length"
        )
654
655
656
657
        out = []
        for i, token_id in enumerate(token_ids):
            logprob = logprobs[i]
            token_logprob = logprob[token_id]
658
659
660
661
662
            text = (
                token_logprob.decoded_token
                if token_logprob.decoded_token is not None
                else tokenizer.decode([token_id])
            )
663
664
665
666
667
            out.append(
                Logprob(
                    token=text,
                    logprob=max(token_logprob.logprob, -9999.0),
                    bytes=list(text.encode("utf-8", errors="replace")),
668
669
670
671
672
673
674
                    top_logprobs=self._topk_logprobs(
                        logprob, top_logprobs=top_logprobs, tokenizer=tokenizer
                    )
                    if top_logprobs
                    else [],
                )
            )
675
676
        return out

677
678
679
    def _create_stream_response_logprobs(
        self,
        token_ids: Sequence[int],
680
        logprobs: SampleLogprobs | None,
681
        tokenizer: AnyTokenizer,
682
        top_logprobs: int | None = None,
683
    ) -> list[response_text_delta_event.Logprob]:
684
685
686
687
688
689
        lgs = self._create_response_logprobs(
            token_ids=token_ids,
            logprobs=logprobs,
            tokenizer=tokenizer,
            top_logprobs=top_logprobs,
        )
690
691
692
693
694
695
        return [
            response_text_delta_event.Logprob(
                token=lg.token,
                logprob=lg.logprob,
                top_logprobs=[
                    response_text_delta_event.LogprobTopLogprob(
696
697
                        token=tl.token, logprob=tl.logprob
                    )
698
                    for tl in lg.top_logprobs
699
700
701
                ],
            )
            for lg in lgs
702
703
        ]

704
705
706
707
708
709
710
711
712
713
714
715
716
    def _make_response_output_items(
        self,
        request: ResponsesRequest,
        final_output: CompletionOutput,
        tokenizer: AnyTokenizer,
    ) -> list[ResponseOutputItem]:
        if self.reasoning_parser:
            try:
                reasoning_parser = self.reasoning_parser(tokenizer)
            except RuntimeError as e:
                logger.exception("Error in reasoning parser creation.")
                raise e

717
718
719
            reasoning_content, content = reasoning_parser.extract_reasoning_content(
                final_output.text, request=request
            )
720
721
722
723
        else:
            reasoning_content = None
            content = final_output.text

724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            output_text = ""
            if content:
                output_text = content
            elif reasoning_content:
                output_text = f"[reasoning: {reasoning_content}]"

            if output_text:
                self.request_logger.log_outputs(
                    request_id=request.request_id,
                    outputs=output_text,
                    output_token_ids=final_output.token_ids,
                    finish_reason=final_output.finish_reason,
                    is_streaming=False,
                    delta=False,
                )

742
743
744
745
746
747
748
        output = []
        if reasoning_content:
            reasoning_item = ResponseReasoningItem(
                id=f"rs_{random_uuid()}",
                summary=[],
                type="reasoning",
                content=[
749
750
751
                    ResponseReasoningTextContent(
                        text=reasoning_content, type="reasoning_text"
                    )
752
753
754
755
756
757
758
759
760
                ],
                status=None,  # NOTE: Only the last output item has status.
            )
            output.append(reasoning_item)
        if content:
            output_text = ResponseOutputText(
                text=content,
                annotations=[],  # TODO
                type="output_text",
761
762
763
764
765
                logprobs=self._create_response_logprobs(
                    token_ids=final_output.token_ids,
                    logprobs=final_output.logprobs,
                    tokenizer=tokenizer,
                    top_logprobs=request.top_logprobs,
766
767
768
                )
                if request.is_include_output_logprobs()
                else None,
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
            )
            message = ResponseOutputMessage(
                id=f"msg_{random_uuid()}",
                content=[output_text],
                role="assistant",
                status="completed",
                type="message",
            )
            output.append(message)
        return output

    def _make_response_output_items_with_harmony(
        self,
        context: HarmonyContext,
    ) -> list[ResponseOutputItem]:
784
        output_items: list[ResponseOutputItem] = []
785
786
787
788
789
790
791
792
793
        num_init_messages = context.num_init_messages
        for msg in context.messages[num_init_messages:]:
            output_items.extend(parse_output_message(msg))
        # Handle the generation stopped in the middle (if any).
        last_items = parse_remaining_state(context.parser)
        if last_items:
            output_items.extend(last_items)
        return output_items

794
795
796
    def _construct_input_messages(
        self,
        request: ResponsesRequest,
797
        prev_response: ResponsesResponse | None = None,
798
799
800
    ) -> list[ChatCompletionMessageParam]:
        messages: list[ChatCompletionMessageParam] = []
        if request.instructions:
801
802
803
804
805
806
            messages.append(
                {
                    "role": "system",
                    "content": request.instructions,
                }
            )
807
808
809
810
811
812
813
814
815
816
817
818

        # Prepend the conversation history.
        if prev_response is not None:
            # Add the previous messages.
            prev_msg = self.msg_store[prev_response.id]
            messages.extend(prev_msg)

            # Add the previous output.
            for output_item in prev_response.output:
                # NOTE: We skip the reasoning output.
                if isinstance(output_item, ResponseOutputMessage):
                    for content in output_item.content:
819
820
821
822
823
824
                        messages.append(
                            {
                                "role": "assistant",
                                "content": content.text,
                            }
                        )
825
826

        # Append the new input.
827
        # Responses API supports simple text inputs without chat format.
828
829
830
831
832
833
        if isinstance(request.input, str):
            messages.append({"role": "user", "content": request.input})
        else:
            messages.extend(request.input)  # type: ignore
        return messages

834
835
836
    def _construct_input_messages_with_harmony(
        self,
        request: ResponsesRequest,
837
        prev_response: ResponsesResponse | None,
838
839
840
841
    ) -> list[OpenAIHarmonyMessage]:
        messages: list[OpenAIHarmonyMessage] = []
        if prev_response is None:
            # New conversation.
842
            reasoning_effort = request.reasoning.effort if request.reasoning else None
843
            tool_types = [tool.type for tool in request.tools]
844
845
846
847
848
849

            # Allow the MCP Tool type to enable built in tools if the
            # server_label is allowlisted in
            # envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS
            if envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS:
                for tool in request.tools:
850
851
852
853
                    if (
                        tool.type == "mcp"
                        and tool.server_label in envs.GPT_OSS_SYSTEM_TOOL_MCP_LABELS
                    ):
854
                        tool_types.append(tool.server_label)
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
            enable_browser = (
                "web_search_preview" in tool_types
                and self.tool_server is not None
                and self.tool_server.has_tool("browser")
            )
            enable_code_interpreter = (
                "code_interpreter" in tool_types
                and self.tool_server is not None
                and self.tool_server.has_tool("python")
            )
            enable_container = (
                "container" in tool_types
                and self.tool_server is not None
                and self.tool_server.has_tool("container")
            )
870
            with_custom_tools = has_custom_tools(tool_types)
871
872
            sys_msg = get_system_message(
                reasoning_effort=reasoning_effort,
873
874
875
876
877
878
879
880
881
                browser_description=self.tool_server.get_tool_description("browser")
                if enable_browser and self.tool_server is not None
                else None,
                python_description=self.tool_server.get_tool_description("python")
                if enable_code_interpreter and self.tool_server is not None
                else None,
                container_description=self.tool_server.get_tool_description("container")
                if enable_container and self.tool_server is not None
                else None,
882
883
                instructions=request.instructions,
                with_custom_tools=with_custom_tools,
884
885
            )
            messages.append(sys_msg)
886
887
            if with_custom_tools:
                dev_msg = get_developer_message(
888
889
                    instructions=request.instructions, tools=request.tools
                )
890
                messages.append(dev_msg)
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
        else:
            # Continue the previous conversation.
            # FIXME(woosuk): Currently, request params like reasoning and
            # instructions are ignored.
            prev_msgs = self.msg_store[prev_response.id]
            # Remove the previous chain-of-thoughts if there is a new "final"
            # message. Note that this also removes these messages from the
            # msg_store.
            if len(prev_msgs) > 0:
                last_msg = prev_msgs[-1]
                assert isinstance(last_msg, OpenAIHarmonyMessage)
                if last_msg.channel == "final":
                    prev_final_msg_idx = -1
                    for i in range(len(prev_msgs) - 2, -1, -1):
                        prev_msg_i = prev_msgs[i]
                        assert isinstance(prev_msg_i, OpenAIHarmonyMessage)
                        if prev_msg_i.channel == "final":
                            prev_final_msg_idx = i
                            break
910
911
                    recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :]
                    del prev_msgs[prev_final_msg_idx + 1 :]
912
913
914
915
916
917
                    for msg in recent_turn_msgs:
                        assert isinstance(msg, OpenAIHarmonyMessage)
                        if msg.channel != "analysis":
                            prev_msgs.append(msg)
            messages.extend(prev_msgs)
        # Append the new input.
co63oc's avatar
co63oc committed
918
        # Responses API supports simple text inputs without chat format.
919
920
921
922
923
924
925
926
        if isinstance(request.input, str):
            messages.append(get_user_message(request.input))
        else:
            if prev_response is not None:
                prev_outputs = copy(prev_response.output)
            else:
                prev_outputs = []
            for response_msg in request.input:
927
                messages.append(parse_response_input(response_msg, prev_outputs))
928
                # User passes in a tool call request and its output. We need
929
930
931
                # to add the tool call request to prev_outputs so that the
                # parse_response_input can find the tool call request when
                # parsing the tool call output.
932
933
934
935
936
                if (
                    isinstance(response_msg, dict)
                    and response_msg.get("type") == "function_call"
                ):
                    response_msg = ResponseFunctionToolCall.model_validate(response_msg)
937
938
939
940
                if isinstance(response_msg, ResponseFunctionToolCall):
                    prev_outputs.append(response_msg)
        return messages

941
942
943
944
945
946
    async def _run_background_request_stream(
        self,
        request: ResponsesRequest,
        *args,
        **kwargs,
    ):
947
        event_deque: deque[StreamingResponsesResponse] = deque()
948
949
950
951
        new_event_signal = asyncio.Event()
        self.event_store[request.request_id] = (event_deque, new_event_signal)
        response = None
        try:
952
            generator = self.responses_stream_generator(request, *args, **kwargs)
953
954
955
956
            async for event in generator:
                event_deque.append(event)
                new_event_signal.set()  # Signal new event available
        except Exception as e:
957
            logger.exception("Background request failed for %s", request.request_id)
958
959
960
961
962
963
964
965
966
967
968
969
970
            response = self.create_error_response(str(e))
        finally:
            new_event_signal.set()

        if response is not None and isinstance(response, ErrorResponse):
            # If the request has failed, update the status to "failed".
            response_id = request.request_id
            async with self.response_store_lock:
                stored_response = self.response_store.get(response_id)
                assert stored_response is not None
                if stored_response.status not in ("completed", "cancelled"):
                    stored_response.status = "failed"

971
972
973
974
975
976
977
    async def _run_background_request(
        self,
        request: ResponsesRequest,
        *args,
        **kwargs,
    ):
        try:
978
            response = await self.responses_full_generator(request, *args, **kwargs)
979
        except Exception as e:
980
            logger.exception("Background request failed for %s", request.request_id)
981
982
983
984
985
986
987
988
989
990
991
            response = self.create_error_response(str(e))

        if isinstance(response, ErrorResponse):
            # If the request has failed, update the status to "failed".
            response_id = request.request_id
            async with self.response_store_lock:
                stored_response = self.response_store.get(response_id)
                assert stored_response is not None
                if stored_response.status not in ("completed", "cancelled"):
                    stored_response.status = "failed"

992
993
994
    async def responses_background_stream_generator(
        self,
        response_id: str,
995
        starting_after: int | None = None,
996
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
        if response_id not in self.event_store:
            raise ValueError(f"Unknown response_id: {response_id}")

        event_deque, new_event_signal = self.event_store[response_id]
        start_index = 0 if starting_after is None else starting_after + 1
        current_index = start_index

        while True:
            new_event_signal.clear()

            # Yield existing events from start_index
            while current_index < len(event_deque):
                event = event_deque[current_index]
                yield event
1011
                if getattr(event, "type", "unknown") == "response.completed":
1012
                    return
1013
1014
1015
1016
                current_index += 1

            await new_event_signal.wait()

1017
1018
1019
    async def retrieve_responses(
        self,
        response_id: str,
1020
1021
1022
1023
1024
1025
1026
        starting_after: int | None,
        stream: bool | None,
    ) -> (
        ErrorResponse
        | ResponsesResponse
        | AsyncGenerator[StreamingResponsesResponse, None]
    ):
1027
1028
1029
1030
1031
        async with self.response_store_lock:
            response = self.response_store.get(response_id)

        if response is None:
            return self._make_not_found_error(response_id)
1032
1033
1034
1035
1036
1037

        if stream:
            return self.responses_background_stream_generator(
                response_id,
                starting_after,
            )
1038
1039
1040
1041
1042
        return response

    async def cancel_responses(
        self,
        response_id: str,
1043
    ) -> ErrorResponse | ResponsesResponse:
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
        async with self.response_store_lock:
            response = self.response_store.get(response_id)
            if response is None:
                return self._make_not_found_error(response_id)

            prev_status = response.status
            if prev_status not in ("queued", "in_progress"):
                return self.create_error_response(
                    err_type="invalid_request_error",
                    message="Cannot cancel a synchronous response.",
                )

            # Update the status to "cancelled".
            response.status = "cancelled"

        # Abort the request.
1060
        if task := self.background_tasks.get(response_id):
1061
1062
1063
1064
            task.cancel()
            try:
                await task
            except asyncio.CancelledError:
1065
                logger.exception("Background task for %s was cancelled", response_id)
1066
1067
1068
1069
1070
1071
1072
1073
        return response

    def _make_not_found_error(self, response_id: str) -> ErrorResponse:
        return self.create_error_response(
            err_type="invalid_request_error",
            message=f"Response with id '{response_id}' not found.",
            status_code=HTTPStatus.NOT_FOUND,
        )
1074
1075
1076
1077

    def _make_store_not_supported_error(self) -> ErrorResponse:
        return self.create_error_response(
            err_type="invalid_request_error",
1078
1079
1080
1081
1082
1083
            message=(
                "`store=True` (default) is not supported. Please set "
                "`store=False` in Responses API or set "
                "`VLLM_ENABLE_RESPONSES_API_STORE=1` in the env var when "
                "starting the vLLM server."
            ),
1084
1085
            status_code=HTTPStatus.BAD_REQUEST,
        )
1086

1087
    async def _process_simple_streaming_events(
1088
1089
1090
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1091
        result_generator: AsyncIterator[ConversationContext | None],
1092
1093
1094
1095
        context: ConversationContext,
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
1096
        created_time: int,
1097
        _increment_sequence_number_and_return: Callable[
1098
1099
            [StreamingResponsesResponse], StreamingResponsesResponse
        ],
1100
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
        current_content_index = 0
        current_output_index = 0
        current_item_id = ""
        reasoning_parser = None
        if self.reasoning_parser:
            reasoning_parser = self.reasoning_parser(tokenizer)
        previous_text = ""
        previous_token_ids: list[int] = []
        first_delta_sent = False
        previous_delta_messages: list[DeltaMessage] = []
        async for ctx in result_generator:
            assert isinstance(ctx, SimpleContext)
            if ctx.last_output is None:
                continue
            if ctx.last_output.outputs:
                output = ctx.last_output.outputs[0]
                if reasoning_parser:
1118
                    delta_message = (
1119
                        reasoning_parser.extract_reasoning_content_streaming(
1120
1121
1122
1123
1124
1125
1126
                            previous_text=previous_text,
                            current_text=previous_text + output.text,
                            delta_text=output.text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=previous_token_ids + output.token_ids,
                            delta_token_ids=output.token_ids,
                        )
1127
1128
                    )
                else:
1129
1130
1131
                    delta_message = DeltaMessage(
                        content=output.text,
                    )
1132
1133
1134
1135
1136
1137
1138
                previous_text += output.text
                previous_token_ids += output.token_ids
                if not delta_message:
                    continue
                if not first_delta_sent:
                    current_item_id = str(uuid.uuid4())
                    if delta_message.reasoning_content:
1139
                        yield _increment_sequence_number_and_return(
1140
1141
1142
1143
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1144
                                item=ResponseReasoningItem(
1145
1146
1147
1148
1149
                                    type="reasoning",
                                    id=current_item_id,
                                    summary=[],
                                    status="in_progress",
                                ),
1150
1151
                            )
                        )
1152
                    else:
1153
                        yield _increment_sequence_number_and_return(
1154
1155
1156
1157
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1158
                                item=ResponseOutputMessage(
1159
1160
1161
1162
1163
1164
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[],
                                    status="in_progress",
                                ),
1165
1166
                            )
                        )
1167
                    yield _increment_sequence_number_and_return(
1168
                        ResponseContentPartAddedEvent(
1169
1170
1171
1172
1173
                            type="response.content_part.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            content_index=current_content_index,
1174
                            part=ResponseOutputText(
1175
1176
1177
1178
1179
                                type="output_text",
                                text="",
                                annotations=[],
                                logprobs=[],
                            ),
1180
1181
                        )
                    )
1182
1183
1184
1185
1186
1187
                    current_content_index += 1
                    first_delta_sent = True
                # todo(kebe7jun) tool call support

                # check delta message and previous delta message are
                # same as content or reasoning content
1188
1189
1190
1191
1192
                if (
                    previous_delta_messages
                    and previous_delta_messages[-1].reasoning_content is not None
                    and delta_message.content is not None
                ):
1193
1194
                    # from reasoning to normal content, send done
                    # event for reasoning
1195
1196
1197
1198
1199
                    reason_content = "".join(
                        pm.reasoning_content
                        for pm in previous_delta_messages
                        if pm.reasoning_content is not None
                    )
1200
                    yield _increment_sequence_number_and_return(
1201
1202
1203
1204
1205
1206
1207
                        ResponseReasoningTextDoneEvent(
                            type="response.reasoning_text.done",
                            item_id=current_item_id,
                            sequence_number=-1,
                            output_index=current_output_index,
                            content_index=current_content_index,
                            text=reason_content,
1208
1209
                        )
                    )
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
                    current_content_index = 0
                    reasoning_item = ResponseReasoningItem(
                        type="reasoning",
                        content=[
                            ResponseReasoningTextContent(
                                text=reason_content,
                                type="reasoning_text",
                            ),
                        ],
                        status="completed",
                        id=current_item_id,
                        summary=[],
                    )
1223
                    yield _increment_sequence_number_and_return(
1224
1225
1226
1227
1228
                        ResponseOutputItemDoneEvent(
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item=reasoning_item,
1229
1230
                        )
                    )
1231
                    yield _increment_sequence_number_and_return(
1232
                        ResponseOutputItemAddedEvent(
1233
1234
1235
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
1236
                            item=ResponseOutputMessage(
1237
1238
1239
1240
1241
1242
                                id=current_item_id,
                                type="message",
                                role="assistant",
                                content=[],
                                status="in_progress",
                            ),
1243
1244
                        )
                    )
1245
1246
                    current_output_index += 1
                    current_item_id = str(uuid.uuid4())
1247
                    yield _increment_sequence_number_and_return(
1248
                        ResponseContentPartAddedEvent(
1249
1250
1251
1252
1253
                            type="response.content_part.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            content_index=current_content_index,
1254
                            part=ResponseOutputText(
1255
1256
1257
1258
1259
                                type="output_text",
                                text="",
                                annotations=[],
                                logprobs=[],
                            ),
1260
1261
                        )
                    )
1262
1263
1264
                    current_content_index += 1
                    # reset previous delta messages
                    previous_delta_messages = []
1265

1266
                if delta_message.reasoning_content is not None:
1267
                    yield _increment_sequence_number_and_return(
1268
1269
1270
1271
1272
1273
1274
                        ResponseReasoningTextDeltaEvent(
                            type="response.reasoning_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=delta_message.reasoning_content,
1275
1276
                        )
                    )
1277
                elif delta_message.content is not None:
1278
                    yield _increment_sequence_number_and_return(
1279
                        ResponseTextDeltaEvent(
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
                            type="response.output_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=delta_message.content,
                            logprobs=self._create_stream_response_logprobs(
                                token_ids=output.token_ids,
                                logprobs=output.logprobs,
                                tokenizer=tokenizer,
                                top_logprobs=request.top_logprobs,
1291
1292
1293
1294
1295
                            )
                            if request.is_include_output_logprobs()
                            else [],
                        )
                    )
1296
1297
1298
1299
1300
                current_content_index += 1

                previous_delta_messages.append(delta_message)
        if previous_delta_messages:
            if previous_delta_messages[-1].reasoning_content is not None:
1301
1302
1303
1304
1305
                reason_content = "".join(
                    pm.reasoning_content
                    for pm in previous_delta_messages
                    if pm.reasoning_content is not None
                )
1306
                yield _increment_sequence_number_and_return(
1307
1308
1309
1310
1311
1312
1313
                    ResponseReasoningTextDoneEvent(
                        type="response.reasoning_text.done",
                        item_id=current_item_id,
                        sequence_number=-1,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        text=reason_content,
1314
1315
                    )
                )
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
                current_content_index += 1
                reasoning_item = ResponseReasoningItem(
                    type="reasoning",
                    content=[
                        ResponseReasoningTextContent(
                            text=reason_content,
                            type="reasoning_text",
                        ),
                    ],
                    status="completed",
                    id=current_item_id,
                    summary=[],
                )
1329
                yield _increment_sequence_number_and_return(
1330
1331
1332
1333
1334
                    ResponseOutputItemDoneEvent(
                        type="response.output_item.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        item=reasoning_item,
1335
1336
                    )
                )
1337
            elif previous_delta_messages[-1].content is not None:
1338
1339
1340
1341
1342
                final_content = "".join(
                    pm.content
                    for pm in previous_delta_messages
                    if pm.content is not None
                )
1343
                yield _increment_sequence_number_and_return(
1344
                    ResponseTextDoneEvent(
1345
1346
1347
1348
1349
1350
1351
                        type="response.output_text.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        text=final_content,
                        logprobs=[],
                        item_id=current_item_id,
1352
1353
                    )
                )
1354
1355
1356
1357
1358
1359
                current_content_index += 1
                part = ResponseOutputText(
                    text=final_content,
                    type="output_text",
                    annotations=[],
                )
1360
                yield _increment_sequence_number_and_return(
1361
                    ResponseContentPartDoneEvent(
1362
1363
1364
1365
1366
1367
                        type="response.content_part.done",
                        sequence_number=-1,
                        item_id=current_item_id,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        part=part,
1368
1369
                    )
                )
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
                current_content_index += 1
                item = ResponseOutputMessage(
                    type="message",
                    role="assistant",
                    content=[
                        part,
                    ],
                    status="completed",
                    id=current_item_id,
                    summary=[],
                )
1381
                yield _increment_sequence_number_and_return(
1382
1383
1384
1385
1386
                    ResponseOutputItemDoneEvent(
                        type="response.output_item.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        item=item,
1387
1388
                    )
                )
1389
1390
1391
1392
1393

    async def _process_harmony_streaming_events(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1394
        result_generator: AsyncIterator[ConversationContext | None],
1395
1396
1397
1398
1399
        context: ConversationContext,
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
        created_time: int,
1400
        _increment_sequence_number_and_return: Callable[
1401
1402
            [StreamingResponsesResponse], StreamingResponsesResponse
        ],
1403
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1404
        current_content_index = -1
1405
        current_output_index = 0
1406
        current_item_id: str = ""
1407
        sent_output_item_added = False
1408
        is_first_function_call_delta = False
1409
1410
1411
1412
1413
1414
        async for ctx in result_generator:
            assert isinstance(ctx, StreamingHarmonyContext)

            if ctx.is_expecting_start():
                current_output_index += 1
                sent_output_item_added = False
1415
                is_first_function_call_delta = False
1416
1417
1418
                if len(ctx.parser.messages) > 0:
                    previous_item = ctx.parser.messages[-1]
                    if previous_item.recipient is not None:
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
                        # Deal with tool call
                        if previous_item.recipient.startswith("functions."):
                            function_name = previous_item.recipient[len("functions.") :]
                            yield _increment_sequence_number_and_return(
                                ResponseFunctionCallArgumentsDoneEvent(
                                    type="response.function_call_arguments.done",
                                    arguments=previous_item.content[0].text,
                                    name=function_name,
                                    item_id=current_item_id,
                                    output_index=current_output_index,
                                    sequence_number=-1,
                                )
                            )
                            function_call_item = ResponseFunctionToolCall(
                                type="function_call",
                                arguments=previous_item.content[0].text,
                                name=function_name,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                sequence_number=-1,
                                call_id=f"fc_{random_uuid()}",
                                status="completed",
                            )
                            yield _increment_sequence_number_and_return(
                                ResponseOutputItemDoneEvent(
                                    type="response.output_item.done",
                                    sequence_number=-1,
                                    output_index=current_output_index,
                                    item=function_call_item,
                                )
                            )
1450
                    elif previous_item.channel == "analysis":
1451
1452
1453
1454
                        content = ResponseReasoningTextContent(
                            text=previous_item.content[0].text,
                            type="reasoning_text",
                        )
1455
1456
                        reasoning_item = ResponseReasoningItem(
                            type="reasoning",
1457
                            content=[content],
1458
                            status="completed",
1459
1460
                            id=current_item_id,
                            summary=[],
1461
                        )
1462
                        yield _increment_sequence_number_and_return(
1463
1464
1465
1466
1467
1468
1469
                            ResponseReasoningTextDoneEvent(
                                type="response.reasoning_text.done",
                                item_id=current_item_id,
                                sequence_number=-1,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                text=previous_item.content[0].text,
1470
1471
                            )
                        )
1472
1473
1474
1475
1476
1477
1478
1479
                        yield _increment_sequence_number_and_return(
                            ResponseReasoningPartDoneEvent(
                                type="response.reasoning_part.done",
                                sequence_number=-1,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                part=content,
1480
1481
                            )
                        )
1482
                        yield _increment_sequence_number_and_return(
1483
1484
1485
1486
1487
                            ResponseOutputItemDoneEvent(
                                type="response.output_item.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item=reasoning_item,
1488
1489
                            )
                        )
1490
1491
1492
1493
1494
1495
                    elif previous_item.channel == "final":
                        text_content = ResponseOutputText(
                            type="output_text",
                            text=previous_item.content[0].text,
                            annotations=[],
                        )
1496
                        yield _increment_sequence_number_and_return(
1497
                            ResponseTextDoneEvent(
1498
1499
1500
1501
1502
1503
1504
                                type="response.output_text.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                text=previous_item.content[0].text,
                                logprobs=[],
                                item_id=current_item_id,
1505
1506
                            )
                        )
1507
                        yield _increment_sequence_number_and_return(
1508
1509
1510
1511
1512
1513
1514
                            ResponseContentPartDoneEvent(
                                type="response.content_part.done",
                                sequence_number=-1,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                part=text_content,
1515
1516
                            )
                        )
1517
                        yield _increment_sequence_number_and_return(
1518
                            ResponseOutputItemDoneEvent(
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
                                type="response.output_item.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item=ResponseOutputMessage(
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[text_content],
                                    status="completed",
                                ),
1529
1530
                            )
                        )
1531

1532
            # stream the output of a harmony message
1533
            if ctx.parser.last_content_delta:
1534
1535
1536
1537
                if (
                    ctx.parser.current_channel == "final"
                    and ctx.parser.current_recipient is None
                ):
1538
1539
                    if not sent_output_item_added:
                        sent_output_item_added = True
1540
                        current_item_id = f"msg_{random_uuid()}"
1541
                        yield _increment_sequence_number_and_return(
1542
1543
1544
1545
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1546
                                item=ResponseOutputMessage(
1547
1548
1549
1550
1551
1552
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[],
                                    status="in_progress",
                                ),
1553
1554
                            )
                        )
1555
                        current_content_index += 1
1556
                        yield _increment_sequence_number_and_return(
1557
1558
1559
1560
1561
1562
                            ResponseContentPartAddedEvent(
                                type="response.content_part.added",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
                                content_index=current_content_index,
1563
                                part=ResponseOutputText(
1564
1565
1566
1567
1568
                                    type="output_text",
                                    text="",
                                    annotations=[],
                                    logprobs=[],
                                ),
1569
1570
                            )
                        )
1571
                    yield _increment_sequence_number_and_return(
1572
                        ResponseTextDeltaEvent(
1573
1574
1575
1576
1577
1578
1579
1580
                            type="response.output_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
                            # TODO, use logprobs from ctx.last_request_output
                            logprobs=[],
1581
1582
1583
1584
1585
1586
                        )
                    )
                elif (
                    ctx.parser.current_channel == "analysis"
                    and ctx.parser.current_recipient is None
                ):
1587
1588
                    if not sent_output_item_added:
                        sent_output_item_added = True
1589
                        current_item_id = f"msg_{random_uuid()}"
1590
                        yield _increment_sequence_number_and_return(
1591
1592
1593
1594
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1595
                                item=ResponseReasoningItem(
1596
1597
1598
1599
1600
                                    type="reasoning",
                                    id=current_item_id,
                                    summary=[],
                                    status="in_progress",
                                ),
1601
1602
                            )
                        )
1603
                        current_content_index += 1
1604
                        yield _increment_sequence_number_and_return(
1605
1606
                            ResponseReasoningPartAddedEvent(
                                type="response.reasoning_part.added",
1607
1608
1609
1610
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
                                content_index=current_content_index,
1611
                                part=ResponseReasoningTextContent(
1612
                                    text="",
1613
                                    type="reasoning_text",
1614
                                ),
1615
1616
                            )
                        )
1617
                    yield _increment_sequence_number_and_return(
1618
1619
1620
1621
1622
1623
1624
                        ResponseReasoningTextDeltaEvent(
                            type="response.reasoning_text.delta",
                            item_id=current_item_id,
                            output_index=current_output_index,
                            content_index=current_content_index,
                            delta=ctx.parser.last_content_delta,
                            sequence_number=-1,
1625
1626
                        )
                    )
1627
1628
1629
                # built-in tools will be triggered on the analysis channel
                # However, occasionally built-in tools will
                # still be output to commentary.
1630
1631
1632
1633
                elif (
                    ctx.parser.current_channel == "commentary"
                    or ctx.parser.current_channel == "analysis"
                ) and ctx.parser.current_recipient == "python":
1634
1635
                    if not sent_output_item_added:
                        sent_output_item_added = True
1636
                        current_item_id = f"tool_{random_uuid()}"
1637
                        yield _increment_sequence_number_and_return(
1638
1639
1640
1641
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1642
                                item=ResponseCodeInterpreterToolCallParam(
1643
1644
1645
1646
1647
1648
1649
                                    type="code_interpreter_call",
                                    id=current_item_id,
                                    code=None,
                                    container_id="auto",
                                    outputs=None,
                                    status="in_progress",
                                ),
1650
1651
                            )
                        )
1652
                        yield _increment_sequence_number_and_return(
1653
                            ResponseCodeInterpreterCallInProgressEvent(
1654
                                type="response.code_interpreter_call.in_progress",
1655
1656
1657
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
1658
1659
                            )
                        )
1660
                    yield _increment_sequence_number_and_return(
1661
1662
1663
1664
1665
1666
                        ResponseCodeInterpreterCallCodeDeltaEvent(
                            type="response.code_interpreter_call_code.delta",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
1667
1668
                        )
                    )
1669
1670

            # stream tool call outputs
1671
1672
            if ctx.is_assistant_action_turn() and len(ctx.parser.messages) > 0:
                previous_item = ctx.parser.messages[-1]
1673
1674
1675
1676
1677
1678
1679
                if (
                    self.tool_server is not None
                    and self.tool_server.has_tool("browser")
                    and previous_item.recipient is not None
                    and previous_item.recipient.startswith("browser.")
                ):
                    function_name = previous_item.recipient[len("browser.") :]
1680
1681
1682
                    action = None
                    parsed_args = json.loads(previous_item.content[0].text)
                    if function_name == "search":
1683
                        action = response_function_web_search.ActionSearch(
1684
1685
                            type="search",
                            query=parsed_args["query"],
1686
                        )
1687
                    elif function_name == "open":
1688
1689
1690
1691
1692
                        action = response_function_web_search.ActionOpenPage(
                            type="open_page",
                            # TODO: translate to url
                            url=f"cursor:{parsed_args.get('cursor', '')}",
                        )
1693
                    elif function_name == "find":
1694
1695
1696
1697
1698
1699
                        action = response_function_web_search.ActionFind(
                            type="find",
                            pattern=parsed_args["pattern"],
                            # TODO: translate to url
                            url=f"cursor:{parsed_args.get('cursor', '')}",
                        )
1700
                    else:
1701
                        raise ValueError(f"Unknown function name: {function_name}")
1702

1703
                    current_item_id = f"tool_{random_uuid()}"
1704
                    yield _increment_sequence_number_and_return(
1705
                        ResponseOutputItemAddedEvent(
1706
1707
1708
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
1709
                            item=response_function_web_search.ResponseFunctionWebSearch(
1710
1711
1712
1713
1714
1715
                                # TODO: generate a unique id for web search call
                                type="web_search_call",
                                id=current_item_id,
                                action=action,
                                status="in_progress",
                            ),
1716
1717
                        )
                    )
1718
                    yield _increment_sequence_number_and_return(
1719
1720
1721
1722
1723
                        ResponseWebSearchCallInProgressEvent(
                            type="response.web_search_call.in_progress",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1724
1725
                        )
                    )
1726
                    yield _increment_sequence_number_and_return(
1727
1728
1729
1730
1731
                        ResponseWebSearchCallSearchingEvent(
                            type="response.web_search_call.searching",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1732
1733
                        )
                    )
1734
1735

                    # enqueue
1736
                    yield _increment_sequence_number_and_return(
1737
1738
1739
1740
1741
                        ResponseWebSearchCallCompletedEvent(
                            type="response.web_search_call.completed",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1742
1743
                        )
                    )
1744
                    yield _increment_sequence_number_and_return(
1745
                        ResponseOutputItemDoneEvent(
1746
1747
1748
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
1749
                            item=ResponseFunctionWebSearch(
1750
1751
1752
1753
1754
                                type="web_search_call",
                                id=current_item_id,
                                action=action,
                                status="completed",
                            ),
1755
1756
                        )
                    )
1757

1758
1759
1760
1761
1762
1763
                if (
                    self.tool_server is not None
                    and self.tool_server.has_tool("python")
                    and previous_item.recipient is not None
                    and previous_item.recipient.startswith("python")
                ):
1764
                    yield _increment_sequence_number_and_return(
1765
1766
1767
1768
1769
                        ResponseCodeInterpreterCallCodeDoneEvent(
                            type="response.code_interpreter_call_code.done",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1770
                            code=previous_item.content[0].text,
1771
1772
                        )
                    )
1773
                    yield _increment_sequence_number_and_return(
1774
1775
1776
1777
1778
                        ResponseCodeInterpreterCallInterpretingEvent(
                            type="response.code_interpreter_call.interpreting",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1779
1780
                        )
                    )
1781
                    yield _increment_sequence_number_and_return(
1782
1783
1784
1785
1786
                        ResponseCodeInterpreterCallCompletedEvent(
                            type="response.code_interpreter_call.completed",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1787
1788
                        )
                    )
1789
                    yield _increment_sequence_number_and_return(
1790
                        ResponseOutputItemDoneEvent(
1791
1792
1793
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
1794
                            item=ResponseCodeInterpreterToolCallParam(
1795
1796
1797
1798
1799
1800
1801
1802
                                type="code_interpreter_call",
                                id=current_item_id,
                                code=previous_item.content[0].text,
                                container_id="auto",
                                # TODO: add outputs here
                                outputs=[],
                                status="completed",
                            ),
1803
1804
                        )
                    )
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
            # developer tools will be triggered on the commentary channel
            # and recipient starts with "functions.TOOL_NAME"
            if (
                ctx.parser.current_channel == "commentary"
                and ctx.parser.current_recipient
                and ctx.parser.current_recipient.startswith("functions.")
            ):
                if is_first_function_call_delta is False:
                    is_first_function_call_delta = True
                    fc_name = ctx.parser.current_recipient[len("functions.") :]
                    tool_call_item = ResponseFunctionToolCall(
                        name=fc_name,
                        type="function_call",
                        id=current_item_id,
                        call_id=f"call_{random_uuid()}",
                        arguments="",
                        status="in_progress",
                    )
                    current_item_id = f"fc_{random_uuid()}"
                    yield _increment_sequence_number_and_return(
                        ResponseOutputItemAddedEvent(
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item=tool_call_item,
                        )
                    )
                else:
                    yield _increment_sequence_number_and_return(
                        ResponseFunctionCallArgumentsDeltaEvent(
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
                            output_index=current_output_index,
                            sequence_number=-1,
                            type="response.function_call_arguments.delta",
                        )
                    )
1842

1843
1844
1845
1846
    async def responses_stream_generator(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1847
        result_generator: AsyncIterator[ConversationContext | None],
1848
1849
1850
1851
        context: ConversationContext,
        model_name: str,
        tokenizer: AnyTokenizer,
        request_metadata: RequestResponseMetadata,
1852
        created_time: int | None = None,
1853
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1854
1855
1856
1857
1858
        # TODO:
        # 1. Handle disconnect

        created_time = created_time or int(time.time())

1859
1860
        sequence_number = 0

1861
        def _increment_sequence_number_and_return(
1862
            event: StreamingResponsesResponse,
1863
        ) -> StreamingResponsesResponse:
1864
1865
            nonlocal sequence_number
            # Set sequence_number if the event has this attribute
1866
            if hasattr(event, "sequence_number"):
1867
1868
                event.sequence_number = sequence_number
            sequence_number += 1
1869
            return event
1870

1871
        async with AsyncExitStack() as exit_stack:
1872
1873
            processer = None
            if self.use_harmony:
1874
1875
                # TODO: in streaming, we noticed this bug:
                # https://github.com/vllm-project/vllm/issues/25697
1876
                await self._initialize_tool_sessions(request, context, exit_stack)
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
                processer = self._process_harmony_streaming_events
            else:
                processer = self._process_simple_streaming_events

            initial_response = ResponsesResponse.from_request(
                request,
                sampling_params,
                model_name=model_name,
                created_time=created_time,
                output=[],
                status="in_progress",
                usage=None,
            ).model_dump()
1890
            yield _increment_sequence_number_and_return(
1891
1892
1893
1894
                ResponseCreatedEvent(
                    type="response.created",
                    sequence_number=-1,
                    response=initial_response,
1895
1896
                )
            )
1897
            yield _increment_sequence_number_and_return(
1898
1899
1900
1901
                ResponseInProgressEvent(
                    type="response.in_progress",
                    sequence_number=-1,
                    response=initial_response,
1902
1903
                )
            )
1904

1905
            async for event_data in processer(
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
                created_time,
                _increment_sequence_number_and_return,
            ):
1916
                yield event_data
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933

            async def empty_async_generator():
                # A hack to trick Python to think this is a generator but
                # in fact it immediately returns.
                if False:
                    yield

            final_response = await self.responses_full_generator(
                request,
                sampling_params,
                empty_async_generator(),
                context,
                model_name,
                tokenizer,
                request_metadata,
                created_time=created_time,
            )
1934
            yield _increment_sequence_number_and_return(
1935
                ResponseCompletedEvent(
1936
1937
                    type="response.completed",
                    sequence_number=-1,
1938
                    response=final_response,
1939
1940
                )
            )