"tests/entrypoints/openai/test_vision.py" did not exist on "dd793d1de59b5efad25f4794b68cb935824c7a11"
serving_responses.py 86.9 KB
Newer Older
1
2
3
4
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import asyncio
5
import json
6
import time
7
import uuid
8
from collections import deque
9
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
10
from contextlib import AsyncExitStack
11
from copy import copy
12
from http import HTTPStatus
13
from typing import Final
14
15
16

import jinja2
from fastapi import Request
17
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
    ResponseCodeInterpreterCallInterpretingEvent,
23
24
25
    ResponseCodeInterpreterToolCallParam,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
26
27
    ResponseFunctionCallArgumentsDeltaEvent,
    ResponseFunctionCallArgumentsDoneEvent,
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    ResponseFunctionToolCall,
    ResponseFunctionWebSearch,
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponseOutputMessage,
    ResponseOutputText,
    ResponseReasoningItem,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseTextDeltaEvent,
    ResponseTextDoneEvent,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
    response_function_web_search,
    response_text_delta_event,
)
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
48
from openai.types.responses.response_reasoning_item import (
49
50
    Content as ResponseReasoningTextContent,
)
51
from openai.types.responses.tool import Mcp, Tool
52
from openai_harmony import Message as OpenAIHarmonyMessage
53
from pydantic import TypeAdapter
54

55
from vllm import envs
56
from vllm.engine.protocol import EngineClient
57
58
59
60
61
62
63
from vllm.entrypoints.chat_utils import (
    ChatCompletionMessageParam,
    ChatTemplateContentFormatOption,
)
from vllm.entrypoints.context import (
    ConversationContext,
    HarmonyContext,
64
    ParsableContext,
65
66
67
    SimpleContext,
    StreamingHarmonyContext,
)
68
69
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.parser.harmony_utils import (
70
    construct_harmony_previous_input_messages,
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_system_message,
    get_user_message,
    has_custom_tools,
    parse_output_message,
    parse_remaining_state,
    parse_response_input,
    render_for_completion,
)
from vllm.entrypoints.openai.protocol import (
    DeltaMessage,
    ErrorResponse,
    InputTokensDetails,
    OutputTokensDetails,
    RequestResponseMetadata,
    ResponseCompletedEvent,
    ResponseCreatedEvent,
    ResponseInProgressEvent,
90
    ResponseInputOutputMessage,
91
92
93
94
95
96
97
    ResponseReasoningPartAddedEvent,
    ResponseReasoningPartDoneEvent,
    ResponsesRequest,
    ResponsesResponse,
    ResponseUsage,
    StreamingResponsesResponse,
)
98
99
100
101
from vllm.entrypoints.openai.serving_engine import (
    GenerationError,
    OpenAIServing,
)
102
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
103
from vllm.entrypoints.responses_utils import (
104
    construct_input_messages,
105
    construct_tool_dicts,
106
107
    extract_tool_types,
)
108
from vllm.entrypoints.tool_server import ToolServer
109
from vllm.inputs.data import TokensPrompt
110
from vllm.logger import init_logger
111
112
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
113
from vllm.outputs import CompletionOutput
114
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
115
from vllm.tokenizers import TokenizerLike
116
117
118
119
120
from vllm.utils import random_uuid

logger = init_logger(__name__)


121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def _extract_allowed_tools_from_mcp_requests(
    tools: list[Tool],
) -> dict[str, list[str] | None]:
    """
    Extract allowed_tools mapping from MCP tool requests.

    Returns a dictionary mapping server_label to allowed_tools list.
    Handles both list format and McpAllowedToolsMcpToolFilter object format.

    Special handling:
    - If allowed_tools is None, returns None (allows all tools)
    - If allowed_tools contains "*", returns None (allows all tools)
    - Otherwise, returns the list of specific tool names

    This function can be reused for both harmony and non-harmony MCP calls.
    """
    allowed_tools_map: dict[str, list[str] | None] = {}
    for tool in tools:
        if not isinstance(tool, Mcp):
            continue

        # allowed_tools can be a list or an object with tool_names
        # Extract the actual list of tool names
        allowed_tools_val = None
        if tool.allowed_tools is not None:
            if isinstance(tool.allowed_tools, list):
                allowed_tools_val = tool.allowed_tools
            elif hasattr(tool.allowed_tools, "tool_names"):
                # It's an McpAllowedToolsMcpToolFilter object
                allowed_tools_val = tool.allowed_tools.tool_names

        # Normalize "*" to None (both mean "allow all tools")
        if allowed_tools_val is not None and "*" in allowed_tools_val:
            allowed_tools_val = None

        allowed_tools_map[tool.server_label] = allowed_tools_val
    return allowed_tools_map


160
161
162
163
164
165
class OpenAIServingResponses(OpenAIServing):
    def __init__(
        self,
        engine_client: EngineClient,
        models: OpenAIServingModels,
        *,
166
167
        request_logger: RequestLogger | None,
        chat_template: str | None,
168
169
170
171
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
        reasoning_parser: str = "",
        enable_auto_tools: bool = False,
172
173
        tool_parser: str | None = None,
        tool_server: ToolServer | None = None,
174
175
        enable_prompt_tokens_details: bool = False,
        enable_force_include_usage: bool = False,
176
        enable_log_outputs: bool = False,
177
        log_error_stack: bool = False,
178
179
180
181
182
183
    ) -> None:
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
184
            log_error_stack=log_error_stack,
185
186
187
188
        )

        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
189
        self.enable_log_outputs = enable_log_outputs
190

191
192
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
193
        )
194
195
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
        self.enable_force_include_usage = enable_force_include_usage
196
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
197
198
199
        if self.default_sampling_params:
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
200
201
202
203
204
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
205

206
207
208
209
210
        # If False (default), the "store" option is (silently) ignored and the
        # response is not stored. If True, the response is stored in memory.
        # NOTE(woosuk): This may not be intuitive for users, as the default
        # behavior in OpenAI's Responses API is to store the response, but
        # vLLM's default behavior is not.
211
        self.enable_store = envs.VLLM_ENABLE_RESPONSES_API_STORE
212
213
214
215
        if self.enable_store:
            logger.warning_once(
                "`VLLM_ENABLE_RESPONSES_API_STORE` is enabled. This may "
                "cause a memory leak since we never remove responses from "
216
217
                "the store."
            )
218

219
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
220
        if self.use_harmony:
221
222
223
224
            logger.warning(
                "For gpt-oss, we ignore --enable-auto-tool-choice "
                "and always enable tool use."
            )
225
226
227
228
229
            # OpenAI models have two EOS-like tokens: <|return|> and <|call|>.
            # We need to add them to the stop token ids.
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
230
231
                get_stop_tokens_for_assistant_actions()
            )
232
        self.enable_auto_tools = enable_auto_tools
233
        # set up tool use
234
235
236
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
        )
237
        # HACK(woosuk): This is a hack. We should use a better store.
238
239
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove responses from the store.
240
241
242
243
        self.response_store: dict[str, ResponsesResponse] = {}
        self.response_store_lock = asyncio.Lock()

        # HACK(woosuk): This is a hack. We should use a better store.
244
245
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove messages from the store.
246
247
        self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}

248
249
250
        # HACK(wuhang): This is a hack. We should use a better store.
        # FIXME: If enable_store=True, this may cause a memory leak since we
        # never remove events from the store.
251
252
253
        self.event_store: dict[
            str, tuple[deque[StreamingResponsesResponse], asyncio.Event]
        ] = {}
254

255
256
        self.background_tasks: dict[str, asyncio.Task] = {}

257
258
        self.tool_server = tool_server

259
    def _validate_generator_input(
260
        self, engine_prompt: TokensPrompt
261
    ) -> ErrorResponse | None:
262
263
264
265
266
267
        """Add validations to the input to the generator here."""
        if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
            error_message = (
                "The engine prompt length"
                f" {len(engine_prompt['prompt_token_ids'])} "
                f"exceeds the max_model_len {self.max_model_len}. "
268
269
                "Please reduce prompt."
            )
270
271
272
273
274
275
276
            return self.create_error_response(
                err_type="invalid_request_error",
                message=error_message,
                status_code=HTTPStatus.BAD_REQUEST,
            )
        return None

277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
    def _validate_create_responses_input(
        self, request: ResponsesRequest
    ) -> ErrorResponse | None:
        if self.use_harmony and request.is_include_output_logprobs():
            return self.create_error_response(
                err_type="invalid_request_error",
                message="logprobs are not supported with gpt-oss models",
                status_code=HTTPStatus.BAD_REQUEST,
            )
        if request.store and not self.enable_store and request.background:
            return self.create_error_response(
                err_type="invalid_request_error",
                message=(
                    "This vLLM engine does not support `store=True` and "
                    "therefore does not support the background mode. To "
                    "enable these features, set the environment variable "
                    "`VLLM_ENABLE_RESPONSES_API_STORE=1` when launching "
                    "the vLLM server."
                ),
                status_code=HTTPStatus.BAD_REQUEST,
            )
298
299
300
301
302
303
304
        if request.previous_input_messages and request.previous_response_id:
            return self.create_error_response(
                err_type="invalid_request_error",
                message="Only one of `previous_input_messages` and "
                "`previous_response_id` can be set.",
                status_code=HTTPStatus.BAD_REQUEST,
            )
305
306
        return None

307
308
309
    async def create_responses(
        self,
        request: ResponsesRequest,
310
311
312
313
314
315
        raw_request: Request | None = None,
    ) -> (
        AsyncGenerator[StreamingResponsesResponse, None]
        | ResponsesResponse
        | ErrorResponse
    ):
316
317
318
319
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
            logger.error("Error with model %s", error_check_ret)
            return error_check_ret
320
321
322
        maybe_validation_error = self._validate_create_responses_input(request)
        if maybe_validation_error is not None:
            return maybe_validation_error
323
324
325
326
327
328
329

        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

330
        if request.store and not self.enable_store:
331
332
333
334
335
336
337
            # Disable the store option.
            # NOTE(woosuk): Although returning an error is possible, we opted
            # to implicitly disable store and process the request anyway, as
            # we assume most users do not intend to actually store the response
            # (i.e., their request's `store=True` just because it's the default
            # value).
            request.store = False
338

339
340
341
342
343
344
345
346
347
348
349
        # Handle the previous response ID.
        prev_response_id = request.previous_response_id
        if prev_response_id is not None:
            async with self.response_store_lock:
                prev_response = self.response_store.get(prev_response_id)
            if prev_response is None:
                return self._make_not_found_error(prev_response_id)
        else:
            prev_response = None

        try:
350
            lora_request = self._maybe_get_adapters(request)
351
            model_name = self.models.model_name(lora_request)
352
            tokenizer = await self.engine_client.get_tokenizer()
353

354
            if self.use_harmony:
355
356
                messages, engine_prompts = self._make_request_with_harmony(
                    request, prev_response
357
                )
358
            else:
359
                messages, engine_prompts = await self._make_request(
360
361
                    request, prev_response, tokenizer
                )
362

363
364
365
366
367
368
369
        except (
            ValueError,
            TypeError,
            RuntimeError,
            jinja2.TemplateError,
            NotImplementedError,
        ) as e:
370
371
372
            logger.exception("Error in preprocessing prompt inputs")
            return self.create_error_response(f"{e} {e.__cause__}")

373
        request_metadata = RequestResponseMetadata(request_id=request.request_id)
374
375
376
377
        if raw_request:
            raw_request.state.request_metadata = request_metadata

        # Schedule the request and get the result generator.
378
        generators: list[AsyncGenerator[ConversationContext, None]] = []
379
380

        builtin_tool_list: list[str] = []
381
        if self.tool_server is not None:
382
383
384
385
            if self.tool_server.has_tool("browser"):
                builtin_tool_list.append("browser")
            if self.tool_server.has_tool("python"):
                builtin_tool_list.append("python")
386
387
            if self.tool_server.has_tool("container"):
                builtin_tool_list.append("container")
388

389
390
391
392
393
394
        if self.tool_server is not None:
            available_tools = builtin_tool_list
        else:
            assert len(builtin_tool_list) == 0
            available_tools = []
        try:
395
            for engine_prompt in engine_prompts:
396
397
398
399
                maybe_error = self._validate_generator_input(engine_prompt)
                if maybe_error is not None:
                    return maybe_error

400
                default_max_tokens = self.max_model_len - len(
401
402
                    engine_prompt["prompt_token_ids"]
                )
403

404
                sampling_params = request.to_sampling_params(
405
406
                    default_max_tokens, self.default_sampling_params
                )
407

408
409
410
411
412
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
413
414
415
416

                context: ConversationContext
                if self.use_harmony:
                    if request.stream:
417
                        context = StreamingHarmonyContext(messages, available_tools)
418
419
420
                    else:
                        context = HarmonyContext(messages, available_tools)
                else:
421
                    if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
422
                        # This is a feature in development for parsing
423
424
425
426
427
428
                        # tokens during generation instead of at the end
                        context = ParsableContext(
                            response_messages=messages,
                            tokenizer=tokenizer,
                            reasoning_parser_cls=self.reasoning_parser,
                            request=request,
429
430
431
432
                            tool_parser_cls=self.tool_parser,
                            available_tools=available_tools,
                            chat_template=self.chat_template,
                            chat_template_content_format=self.chat_template_content_format,
433
434
435
                        )
                    else:
                        context = SimpleContext()
436
437
438
439
440
441
442
443
444
445
446
447
448

                if self.reasoning_parser is not None:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                    if sampling_params.structured_outputs is None:
                        sampling_params.structured_outputs = StructuredOutputsParams()
                    struct_out = sampling_params.structured_outputs
                    if struct_out.all_non_structural_tag_constraints_none():
                        sampling_params.structured_outputs.structural_tag = (
                            reasoning_parser.prepare_structured_tag(
                                sampling_params.structured_outputs.structural_tag,
                                self.tool_server,
                            )
                        )
449
450
451
452
453
454
455
456
                generator = self._generate_with_builtin_tools(
                    request_id=request.request_id,
                    engine_prompt=engine_prompt,
                    sampling_params=sampling_params,
                    context=context,
                    lora_request=lora_request,
                    priority=request.priority,
                    trace_headers=trace_headers,
457
                )
458
459
460
461
                generators.append(generator)
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
462

463
        assert len(generators) == 1
464
        (result_generator,) = generators
465
466
467
468

        # Store the input messages.
        if request.store:
            self.msg_store[request.request_id] = messages
469

470
471
472
473
474
475
476
477
478
479
480
481
482
        if request.background:
            created_time = int(time.time())
            response = ResponsesResponse.from_request(
                request,
                sampling_params,
                model_name=model_name,
                created_time=created_time,
                output=[],
                status="queued",
                usage=None,
            )
            async with self.response_store_lock:
                self.response_store[response.id] = response
483

484
            # Run the request in the background.
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
            if request.stream:
                task = asyncio.create_task(
                    self._run_background_request_stream(
                        request,
                        sampling_params,
                        result_generator,
                        context,
                        model_name,
                        tokenizer,
                        request_metadata,
                        created_time,
                    ),
                    name=f"create_{request.request_id}",
                )
            else:
                task = asyncio.create_task(
                    self._run_background_request(
                        request,
                        sampling_params,
                        result_generator,
                        context,
                        model_name,
                        tokenizer,
                        request_metadata,
                        created_time,
                    ),
                    name=f"create_{response.id}",
                )
513

514
515
516
517
            # For cleanup.
            response_id = response.id
            self.background_tasks[response_id] = task
            task.add_done_callback(
518
519
                lambda _: self.background_tasks.pop(response_id, None)
            )
520
521

            if request.stream:
522
                return self.responses_background_stream_generator(request.request_id)
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
            return response

        if request.stream:
            return self.responses_stream_generator(
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
            )

        try:
            return await self.responses_full_generator(
                request,
                sampling_params,
                result_generator,
                context,
                model_name,
                tokenizer,
                request_metadata,
            )
546
547
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
548
549
        except Exception as e:
            return self.create_error_response(str(e))
550

551
552
553
    async def _make_request(
        self,
        request: ResponsesRequest,
554
        prev_response: ResponsesResponse | None,
555
        tokenizer: TokenizerLike,
556
    ):
557
        tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
558
        # Construct the input messages.
559
560
561
562
563
564
        messages = construct_input_messages(
            request_instructions=request.instructions,
            request_input=request.input,
            prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
            prev_response_output=prev_response.output if prev_response else None,
        )
565
        _, engine_prompts = await self._preprocess_chat(
566
567
568
            request,
            tokenizer,
            messages,
569
570
            tool_dicts=tool_dicts,
            tool_parser=self.tool_parser,
571
572
573
            chat_template=self.chat_template,
            chat_template_content_format=self.chat_template_content_format,
        )
574
        return messages, engine_prompts
575
576
577
578

    def _make_request_with_harmony(
        self,
        request: ResponsesRequest,
579
        prev_response: ResponsesResponse | None,
580
581
582
    ):
        if request.tool_choice != "auto":
            raise NotImplementedError(
583
584
585
                "Only 'auto' tool_choice is supported in response API with Harmony"
            )
        messages = self._construct_input_messages_with_harmony(request, prev_response)
586
        prompt_token_ids = render_for_completion(messages)
587
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
588
589
590
591
592

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

593
        return messages, [engine_prompt]
594

595
596
597
598
599
600
    async def _initialize_tool_sessions(
        self,
        request: ResponsesRequest,
        context: ConversationContext,
        exit_stack: AsyncExitStack,
    ):
601
602
603
604
        # we should only initialize the tool session if the request needs tools
        if len(request.tools) == 0:
            return
        mcp_tools = {
605
            tool.server_label: tool for tool in request.tools if tool.type == "mcp"
606
        }
607
608
609
        await context.init_tool_sessions(
            self.tool_server, exit_stack, request.request_id, mcp_tools
        )
610

611
612
613
614
    async def responses_full_generator(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
615
        result_generator: AsyncIterator[ConversationContext],
616
        context: ConversationContext,
617
        model_name: str,
618
        tokenizer: TokenizerLike,
619
        request_metadata: RequestResponseMetadata,
620
621
        created_time: int | None = None,
    ) -> ErrorResponse | ResponsesResponse:
622
623
624
        if created_time is None:
            created_time = int(time.time())

625
626
        async with AsyncExitStack() as exit_stack:
            try:
627
                await self._initialize_tool_sessions(request, context, exit_stack)
628
629
630
631
632
633
634
                async for _ in result_generator:
                    pass
            except asyncio.CancelledError:
                return self.create_error_response("Client disconnected")
            except ValueError as e:
                # TODO: Use a vllm-specific Validation Error
                return self.create_error_response(str(e))
635

636
637
638
639
640
        # NOTE: Implementation of stauts is still WIP, but for now
        # we guarantee that if the status is not "completed", it is accurate.
        # "completed" is implemented as the "catch-all" for now.
        status: ResponseStatus = "completed"

641
642
        input_messages: ResponseInputOutputMessage | None = None
        output_messages: ResponseInputOutputMessage | None = None
643
644
645
        if self.use_harmony:
            assert isinstance(context, HarmonyContext)
            output = self._make_response_output_items_with_harmony(context)
646
            if request.enable_response_messages:
647
648
                input_messages = context.messages[: context.num_init_messages]
                output_messages = context.messages[context.num_init_messages :]
649
            num_tool_output_tokens = context.num_tool_output_tokens
650
651
652
653
654
            if len(output) > 0:
                if context.finish_reason == "length":
                    status = "incomplete"
                elif context.finish_reason == "abort":
                    status = "cancelled"
655
656
                else:
                    self._raise_if_error(context.finish_reason, request.request_id)
657
658
            else:
                status = "incomplete"
659
        elif isinstance(context, ParsableContext):
660
            output = context.parser.make_response_output_items_from_parsable_context()
661
662

            if request.enable_response_messages:
663
664
                input_messages = context.input_messages
                output_messages = context.output_messages
665
666
667
668

            # TODO: Calculate usage.
            # assert final_res.prompt_token_ids is not None
            num_tool_output_tokens = 0
669
        else:
670
            assert isinstance(context, SimpleContext)
671
672
            # Use final_output which has accumulated text/token_ids/logprobs
            final_res = context.final_output
673
674
675
676
            assert final_res is not None
            assert len(final_res.outputs) == 1
            final_output = final_res.outputs[0]

677
678
679
            # finish_reason='error' indicates retryable internal error
            self._raise_if_error(final_output.finish_reason, request.request_id)

680
            output = self._make_response_output_items(request, final_output, tokenizer)
681

682
            if request.enable_response_messages:
683
684
685
                input_messages = context.input_messages
                output_messages = context.output_messages

686
687
            # Calculate usage.
            assert final_res.prompt_token_ids is not None
688
689
            num_tool_output_tokens = 0

690
        assert isinstance(context, (SimpleContext, HarmonyContext, ParsableContext))
691
692
693
694
        num_prompt_tokens = context.num_prompt_tokens
        num_generated_tokens = context.num_output_tokens
        num_cached_tokens = context.num_cached_tokens
        num_reasoning_tokens = context.num_reasoning_tokens
695
696
697
698

        usage = ResponseUsage(
            input_tokens=num_prompt_tokens,
            output_tokens=num_generated_tokens,
699
            total_tokens=num_prompt_tokens + num_generated_tokens,
700
701
702
703
704
705
706
707
708
            input_tokens_details=InputTokensDetails(
                cached_tokens=num_cached_tokens,
                input_tokens_per_turn=[
                    turn.input_tokens for turn in context.all_turn_metrics
                ],
                cached_tokens_per_turn=[
                    turn.cached_input_tokens for turn in context.all_turn_metrics
                ],
            ),
709
            output_tokens_details=OutputTokensDetails(
710
                reasoning_tokens=num_reasoning_tokens,
711
                tool_output_tokens=num_tool_output_tokens,
712
713
714
715
716
717
                output_tokens_per_turn=[
                    turn.output_tokens for turn in context.all_turn_metrics
                ],
                tool_output_tokens_per_turn=[
                    turn.tool_output_tokens for turn in context.all_turn_metrics
                ],
718
            ),
719
720
721
722
        )
        response = ResponsesResponse.from_request(
            request,
            sampling_params,
723
724
            input_messages=input_messages,
            output_messages=output_messages,
725
726
727
            model_name=model_name,
            created_time=created_time,
            output=output,
728
            status=status,
729
730
731
732
733
734
735
            usage=usage,
        )

        if request.store:
            async with self.response_store_lock:
                stored_response = self.response_store.get(response.id)
                # If the response is already cancelled, don't update it.
736
                if stored_response is None or stored_response.status != "cancelled":
737
738
739
                    self.response_store[response.id] = response
        return response

740
741
742
743
    def _topk_logprobs(
        self,
        logprobs: dict[int, SampleLogprob],
        top_logprobs: int,
744
        tokenizer: TokenizerLike,
745
    ) -> list[LogprobTopLogprob]:
746
747
748
749
750
        """Returns the top-k logprobs from the logprobs dictionary."""
        out = []
        for i, (token_id, _logprob) in enumerate(logprobs.items()):
            if i >= top_logprobs:
                break
751
752
753
754
755
            text = (
                _logprob.decoded_token
                if _logprob.decoded_token is not None
                else tokenizer.decode([token_id])
            )
756
757
758
759
760
            out.append(
                LogprobTopLogprob(
                    token=text,
                    logprob=max(_logprob.logprob, -9999.0),
                    bytes=list(text.encode("utf-8", errors="replace")),
761
762
                )
            )
763
764
765
        return out

    def _create_response_logprobs(
766
767
        self,
        token_ids: Sequence[int],
768
        logprobs: SampleLogprobs | None,
769
        tokenizer: TokenizerLike,
770
        top_logprobs: int | None = None,
771
    ) -> list[Logprob]:
772
773
        assert logprobs is not None, "logprobs must be provided"
        assert len(token_ids) == len(logprobs), (
774
775
            "token_ids and logprobs.token_ids must have the same length"
        )
776
777
778
779
        out = []
        for i, token_id in enumerate(token_ids):
            logprob = logprobs[i]
            token_logprob = logprob[token_id]
780
781
782
783
784
            text = (
                token_logprob.decoded_token
                if token_logprob.decoded_token is not None
                else tokenizer.decode([token_id])
            )
785
786
787
788
789
            out.append(
                Logprob(
                    token=text,
                    logprob=max(token_logprob.logprob, -9999.0),
                    bytes=list(text.encode("utf-8", errors="replace")),
790
791
792
793
794
795
796
                    top_logprobs=(
                        self._topk_logprobs(
                            logprob, top_logprobs=top_logprobs, tokenizer=tokenizer
                        )
                        if top_logprobs
                        else []
                    ),
797
798
                )
            )
799
800
        return out

801
802
803
    def _create_stream_response_logprobs(
        self,
        token_ids: Sequence[int],
804
        logprobs: SampleLogprobs | None,
805
        tokenizer: TokenizerLike,
806
        top_logprobs: int | None = None,
807
    ) -> list[response_text_delta_event.Logprob]:
808
809
810
811
812
813
        lgs = self._create_response_logprobs(
            token_ids=token_ids,
            logprobs=logprobs,
            tokenizer=tokenizer,
            top_logprobs=top_logprobs,
        )
814
815
816
817
818
819
        return [
            response_text_delta_event.Logprob(
                token=lg.token,
                logprob=lg.logprob,
                top_logprobs=[
                    response_text_delta_event.LogprobTopLogprob(
820
821
                        token=tl.token, logprob=tl.logprob
                    )
822
                    for tl in lg.top_logprobs
823
824
825
                ],
            )
            for lg in lgs
826
827
        ]

828
829
830
831
    def _make_response_output_items(
        self,
        request: ResponsesRequest,
        final_output: CompletionOutput,
832
        tokenizer: TokenizerLike,
833
834
835
836
837
838
839
840
    ) -> list[ResponseOutputItem]:
        if self.reasoning_parser:
            try:
                reasoning_parser = self.reasoning_parser(tokenizer)
            except RuntimeError as e:
                logger.exception("Error in reasoning parser creation.")
                raise e

841
            reasoning, content = reasoning_parser.extract_reasoning(
842
843
                final_output.text, request=request
            )
844
        else:
845
            reasoning = None
846
847
            content = final_output.text

848
849
850
851
852
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            output_text = ""
            if content:
                output_text = content
853
854
            elif reasoning:
                output_text = f"[reasoning: {reasoning}]"
855
856
857
858
859
860
861
862
863
864
865

            if output_text:
                self.request_logger.log_outputs(
                    request_id=request.request_id,
                    outputs=output_text,
                    output_token_ids=final_output.token_ids,
                    finish_reason=final_output.finish_reason,
                    is_streaming=False,
                    delta=False,
                )

866
867
        reasoning_item = None
        message_item = None
868
        if reasoning:
869
870
871
872
873
            reasoning_item = ResponseReasoningItem(
                id=f"rs_{random_uuid()}",
                summary=[],
                type="reasoning",
                content=[
874
                    ResponseReasoningTextContent(text=reasoning, type="reasoning_text")
875
876
877
                ],
                status=None,  # NOTE: Only the last output item has status.
            )
878
879
880
881
882
883
884
        tool_calls, content = self._parse_tool_calls_from_content(
            request=request,
            tokenizer=tokenizer,
            content=content,
            enable_auto_tools=self.enable_auto_tools,
            tool_parser_cls=self.tool_parser,
        )
885
886
887
888
889
        if content:
            output_text = ResponseOutputText(
                text=content,
                annotations=[],  # TODO
                type="output_text",
890
891
892
893
894
895
896
897
898
899
                logprobs=(
                    self._create_response_logprobs(
                        token_ids=final_output.token_ids,
                        logprobs=final_output.logprobs,
                        tokenizer=tokenizer,
                        top_logprobs=request.top_logprobs,
                    )
                    if request.is_include_output_logprobs()
                    else None
                ),
900
            )
901
            message_item = ResponseOutputMessage(
902
903
904
905
906
907
                id=f"msg_{random_uuid()}",
                content=[output_text],
                role="assistant",
                status="completed",
                type="message",
            )
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
        outputs = []

        if reasoning_item:
            outputs.append(reasoning_item)
        if message_item:
            outputs.append(message_item)
        if tool_calls:
            tool_call_items = [
                ResponseFunctionToolCall(
                    id=f"fc_{random_uuid()}",
                    call_id=f"call_{random_uuid()}",
                    type="function_call",
                    status="completed",
                    name=tool_call.name,
                    arguments=tool_call.arguments,
                )
                for tool_call in tool_calls
            ]
            outputs.extend(tool_call_items)
        return outputs
928
929
930
931
932

    def _make_response_output_items_with_harmony(
        self,
        context: HarmonyContext,
    ) -> list[ResponseOutputItem]:
933
        output_items: list[ResponseOutputItem] = []
934
935
936
937
938
939
940
941
942
        num_init_messages = context.num_init_messages
        for msg in context.messages[num_init_messages:]:
            output_items.extend(parse_output_message(msg))
        # Handle the generation stopped in the middle (if any).
        last_items = parse_remaining_state(context.parser)
        if last_items:
            output_items.extend(last_items)
        return output_items

943
    def _construct_harmony_system_input_message(
944
        self, request: ResponsesRequest, with_custom_tools: bool, tool_types: set[str]
945
946
    ) -> OpenAIHarmonyMessage:
        reasoning_effort = request.reasoning.effort if request.reasoning else None
947
948
949
950
951
952
953
954
955
956
957

        # Extract allowed_tools from MCP tool requests
        allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)

        # Get filtered tool descriptions first.
        # If get_tool_description returns None (due to filtering), the tool is disabled.
        browser_description = (
            self.tool_server.get_tool_description(
                "browser", allowed_tools_map.get("web_search_preview")
            )
            if "web_search_preview" in tool_types
958
959
            and self.tool_server is not None
            and self.tool_server.has_tool("browser")
960
            else None
961
        )
962
963
964
965
966
        python_description = (
            self.tool_server.get_tool_description(
                "python", allowed_tools_map.get("code_interpreter")
            )
            if "code_interpreter" in tool_types
967
968
            and self.tool_server is not None
            and self.tool_server.has_tool("python")
969
            else None
970
        )
971
972
973
974
975
        container_description = (
            self.tool_server.get_tool_description(
                "container", allowed_tools_map.get("container")
            )
            if "container" in tool_types
976
977
            and self.tool_server is not None
            and self.tool_server.has_tool("container")
978
            else None
979
        )
980

981
982
        sys_msg = get_system_message(
            reasoning_effort=reasoning_effort,
983
984
985
            browser_description=browser_description,
            python_description=python_description,
            container_description=container_description,
986
987
988
989
990
            instructions=request.instructions,
            with_custom_tools=with_custom_tools,
        )
        return sys_msg

991
992
993
    def _construct_input_messages_with_harmony(
        self,
        request: ResponsesRequest,
994
        prev_response: ResponsesResponse | None,
995
996
997
998
    ) -> list[OpenAIHarmonyMessage]:
        messages: list[OpenAIHarmonyMessage] = []
        if prev_response is None:
            # New conversation.
999
            tool_types = extract_tool_types(request.tools)
1000
            with_custom_tools = has_custom_tools(tool_types)
1001
1002
1003

            sys_msg = self._construct_harmony_system_input_message(
                request, with_custom_tools, tool_types
1004
1005
            )
            messages.append(sys_msg)
1006
1007
            if with_custom_tools:
                dev_msg = get_developer_message(
1008
1009
                    instructions=request.instructions, tools=request.tools
                )
1010
                messages.append(dev_msg)
1011
1012
            messages += construct_harmony_previous_input_messages(request)

1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
        else:
            # Continue the previous conversation.
            # FIXME(woosuk): Currently, request params like reasoning and
            # instructions are ignored.
            prev_msgs = self.msg_store[prev_response.id]
            # Remove the previous chain-of-thoughts if there is a new "final"
            # message. Note that this also removes these messages from the
            # msg_store.
            if len(prev_msgs) > 0:
                last_msg = prev_msgs[-1]
                assert isinstance(last_msg, OpenAIHarmonyMessage)
                if last_msg.channel == "final":
                    prev_final_msg_idx = -1
                    for i in range(len(prev_msgs) - 2, -1, -1):
                        prev_msg_i = prev_msgs[i]
                        assert isinstance(prev_msg_i, OpenAIHarmonyMessage)
                        if prev_msg_i.channel == "final":
                            prev_final_msg_idx = i
                            break
1032
1033
                    recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :]
                    del prev_msgs[prev_final_msg_idx + 1 :]
1034
1035
1036
1037
1038
1039
                    for msg in recent_turn_msgs:
                        assert isinstance(msg, OpenAIHarmonyMessage)
                        if msg.channel != "analysis":
                            prev_msgs.append(msg)
            messages.extend(prev_msgs)
        # Append the new input.
co63oc's avatar
co63oc committed
1040
        # Responses API supports simple text inputs without chat format.
1041
1042
1043
1044
1045
1046
1047
1048
        if isinstance(request.input, str):
            messages.append(get_user_message(request.input))
        else:
            if prev_response is not None:
                prev_outputs = copy(prev_response.output)
            else:
                prev_outputs = []
            for response_msg in request.input:
1049
                messages.append(parse_response_input(response_msg, prev_outputs))
1050
                # User passes in a tool call request and its output. We need
1051
1052
1053
1054
1055
1056
1057
                # to add the tool call request to prev_outputs so that the
                # parse_response_input can find the tool call request when
                # parsing the tool call output.
                if isinstance(response_msg, ResponseFunctionToolCall):
                    prev_outputs.append(response_msg)
        return messages

1058
1059
1060
1061
1062
1063
    async def _run_background_request_stream(
        self,
        request: ResponsesRequest,
        *args,
        **kwargs,
    ):
1064
        event_deque: deque[StreamingResponsesResponse] = deque()
1065
1066
1067
1068
        new_event_signal = asyncio.Event()
        self.event_store[request.request_id] = (event_deque, new_event_signal)
        response = None
        try:
1069
            generator = self.responses_stream_generator(request, *args, **kwargs)
1070
1071
1072
            async for event in generator:
                event_deque.append(event)
                new_event_signal.set()  # Signal new event available
1073
1074
        except GenerationError as e:
            response = self._convert_generation_error_to_response(e)
1075
        except Exception as e:
1076
            logger.exception("Background request failed for %s", request.request_id)
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
            response = self.create_error_response(str(e))
        finally:
            new_event_signal.set()

        if response is not None and isinstance(response, ErrorResponse):
            # If the request has failed, update the status to "failed".
            response_id = request.request_id
            async with self.response_store_lock:
                stored_response = self.response_store.get(response_id)
                assert stored_response is not None
                if stored_response.status not in ("completed", "cancelled"):
                    stored_response.status = "failed"

1090
1091
1092
1093
1094
1095
1096
    async def _run_background_request(
        self,
        request: ResponsesRequest,
        *args,
        **kwargs,
    ):
        try:
1097
            response = await self.responses_full_generator(request, *args, **kwargs)
1098
1099
        except GenerationError as e:
            response = self._convert_generation_error_to_response(e)
1100
        except Exception as e:
1101
            logger.exception("Background request failed for %s", request.request_id)
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
            response = self.create_error_response(str(e))

        if isinstance(response, ErrorResponse):
            # If the request has failed, update the status to "failed".
            response_id = request.request_id
            async with self.response_store_lock:
                stored_response = self.response_store.get(response_id)
                assert stored_response is not None
                if stored_response.status not in ("completed", "cancelled"):
                    stored_response.status = "failed"

1113
1114
1115
    async def responses_background_stream_generator(
        self,
        response_id: str,
1116
        starting_after: int | None = None,
1117
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
        if response_id not in self.event_store:
            raise ValueError(f"Unknown response_id: {response_id}")

        event_deque, new_event_signal = self.event_store[response_id]
        start_index = 0 if starting_after is None else starting_after + 1
        current_index = start_index

        while True:
            new_event_signal.clear()

            # Yield existing events from start_index
            while current_index < len(event_deque):
                event = event_deque[current_index]
                yield event
1132
                if getattr(event, "type", "unknown") == "response.completed":
1133
                    return
1134
1135
1136
1137
                current_index += 1

            await new_event_signal.wait()

1138
1139
1140
    async def retrieve_responses(
        self,
        response_id: str,
1141
1142
1143
1144
1145
1146
1147
        starting_after: int | None,
        stream: bool | None,
    ) -> (
        ErrorResponse
        | ResponsesResponse
        | AsyncGenerator[StreamingResponsesResponse, None]
    ):
1148
1149
1150
1151
1152
        async with self.response_store_lock:
            response = self.response_store.get(response_id)

        if response is None:
            return self._make_not_found_error(response_id)
1153
1154
1155
1156
1157
1158

        if stream:
            return self.responses_background_stream_generator(
                response_id,
                starting_after,
            )
1159
1160
1161
1162
1163
        return response

    async def cancel_responses(
        self,
        response_id: str,
1164
    ) -> ErrorResponse | ResponsesResponse:
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
        async with self.response_store_lock:
            response = self.response_store.get(response_id)
            if response is None:
                return self._make_not_found_error(response_id)

            prev_status = response.status
            if prev_status not in ("queued", "in_progress"):
                return self.create_error_response(
                    err_type="invalid_request_error",
                    message="Cannot cancel a synchronous response.",
                )

            # Update the status to "cancelled".
            response.status = "cancelled"

        # Abort the request.
1181
        if task := self.background_tasks.get(response_id):
1182
1183
1184
1185
            task.cancel()
            try:
                await task
            except asyncio.CancelledError:
1186
                logger.exception("Background task for %s was cancelled", response_id)
1187
1188
1189
1190
1191
1192
1193
1194
        return response

    def _make_not_found_error(self, response_id: str) -> ErrorResponse:
        return self.create_error_response(
            err_type="invalid_request_error",
            message=f"Response with id '{response_id}' not found.",
            status_code=HTTPStatus.NOT_FOUND,
        )
1195
1196
1197
1198

    def _make_store_not_supported_error(self) -> ErrorResponse:
        return self.create_error_response(
            err_type="invalid_request_error",
1199
1200
1201
1202
1203
1204
            message=(
                "`store=True` (default) is not supported. Please set "
                "`store=False` in Responses API or set "
                "`VLLM_ENABLE_RESPONSES_API_STORE=1` in the env var when "
                "starting the vLLM server."
            ),
1205
1206
            status_code=HTTPStatus.BAD_REQUEST,
        )
1207

1208
    async def _process_simple_streaming_events(
1209
1210
1211
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1212
        result_generator: AsyncIterator[ConversationContext | None],
1213
1214
        context: ConversationContext,
        model_name: str,
1215
        tokenizer: TokenizerLike,
1216
        request_metadata: RequestResponseMetadata,
1217
        created_time: int,
1218
        _increment_sequence_number_and_return: Callable[
1219
1220
            [StreamingResponsesResponse], StreamingResponsesResponse
        ],
1221
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
        current_content_index = 0
        current_output_index = 0
        current_item_id = ""
        reasoning_parser = None
        if self.reasoning_parser:
            reasoning_parser = self.reasoning_parser(tokenizer)
        previous_text = ""
        previous_token_ids: list[int] = []
        first_delta_sent = False
        previous_delta_messages: list[DeltaMessage] = []
        async for ctx in result_generator:
            assert isinstance(ctx, SimpleContext)
            if ctx.last_output is None:
                continue
            if ctx.last_output.outputs:
                output = ctx.last_output.outputs[0]
1238
1239
                # finish_reason='error' indicates a retryable error
                self._raise_if_error(output.finish_reason, request.request_id)
1240
                if reasoning_parser:
1241
1242
1243
1244
1245
1246
1247
                    delta_message = reasoning_parser.extract_reasoning_streaming(
                        previous_text=previous_text,
                        current_text=previous_text + output.text,
                        delta_text=output.text,
                        previous_token_ids=previous_token_ids,
                        current_token_ids=previous_token_ids + output.token_ids,
                        delta_token_ids=output.token_ids,
1248
1249
                    )
                else:
1250
1251
1252
                    delta_message = DeltaMessage(
                        content=output.text,
                    )
1253
1254
1255
1256
1257
1258
                previous_text += output.text
                previous_token_ids += output.token_ids
                if not delta_message:
                    continue
                if not first_delta_sent:
                    current_item_id = str(uuid.uuid4())
1259
                    if delta_message.reasoning:
1260
                        yield _increment_sequence_number_and_return(
1261
1262
1263
1264
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1265
                                item=ResponseReasoningItem(
1266
1267
1268
1269
1270
                                    type="reasoning",
                                    id=current_item_id,
                                    summary=[],
                                    status="in_progress",
                                ),
1271
1272
                            )
                        )
1273
                    else:
1274
                        yield _increment_sequence_number_and_return(
1275
1276
1277
1278
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1279
                                item=ResponseOutputMessage(
1280
1281
1282
1283
1284
1285
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[],
                                    status="in_progress",
                                ),
1286
1287
                            )
                        )
1288
                    yield _increment_sequence_number_and_return(
1289
                        ResponseContentPartAddedEvent(
1290
1291
1292
1293
1294
                            type="response.content_part.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            content_index=current_content_index,
1295
                            part=ResponseOutputText(
1296
1297
1298
1299
1300
                                type="output_text",
                                text="",
                                annotations=[],
                                logprobs=[],
                            ),
1301
1302
                        )
                    )
1303
1304
1305
1306
1307
1308
                    current_content_index += 1
                    first_delta_sent = True
                # todo(kebe7jun) tool call support

                # check delta message and previous delta message are
                # same as content or reasoning content
1309
1310
                if (
                    previous_delta_messages
1311
                    and previous_delta_messages[-1].reasoning is not None
1312
1313
                    and delta_message.content is not None
                ):
1314
1315
                    # from reasoning to normal content, send done
                    # event for reasoning
1316
                    reason_content = "".join(
1317
                        pm.reasoning
1318
                        for pm in previous_delta_messages
1319
                        if pm.reasoning is not None
1320
                    )
1321
                    yield _increment_sequence_number_and_return(
1322
1323
1324
1325
1326
1327
1328
                        ResponseReasoningTextDoneEvent(
                            type="response.reasoning_text.done",
                            item_id=current_item_id,
                            sequence_number=-1,
                            output_index=current_output_index,
                            content_index=current_content_index,
                            text=reason_content,
1329
1330
                        )
                    )
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
                    current_content_index = 0
                    reasoning_item = ResponseReasoningItem(
                        type="reasoning",
                        content=[
                            ResponseReasoningTextContent(
                                text=reason_content,
                                type="reasoning_text",
                            ),
                        ],
                        status="completed",
                        id=current_item_id,
                        summary=[],
                    )
1344
                    yield _increment_sequence_number_and_return(
1345
1346
1347
1348
1349
                        ResponseOutputItemDoneEvent(
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item=reasoning_item,
1350
1351
                        )
                    )
1352
                    yield _increment_sequence_number_and_return(
1353
                        ResponseOutputItemAddedEvent(
1354
1355
1356
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
1357
                            item=ResponseOutputMessage(
1358
1359
1360
1361
1362
1363
                                id=current_item_id,
                                type="message",
                                role="assistant",
                                content=[],
                                status="in_progress",
                            ),
1364
1365
                        )
                    )
1366
1367
                    current_output_index += 1
                    current_item_id = str(uuid.uuid4())
1368
                    yield _increment_sequence_number_and_return(
1369
                        ResponseContentPartAddedEvent(
1370
1371
1372
1373
1374
                            type="response.content_part.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            content_index=current_content_index,
1375
                            part=ResponseOutputText(
1376
1377
1378
1379
1380
                                type="output_text",
                                text="",
                                annotations=[],
                                logprobs=[],
                            ),
1381
1382
                        )
                    )
1383
1384
1385
                    current_content_index += 1
                    # reset previous delta messages
                    previous_delta_messages = []
1386

1387
                if delta_message.reasoning is not None:
1388
                    yield _increment_sequence_number_and_return(
1389
1390
1391
1392
1393
1394
                        ResponseReasoningTextDeltaEvent(
                            type="response.reasoning_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
1395
                            delta=delta_message.reasoning,
1396
1397
                        )
                    )
1398
                elif delta_message.content is not None:
1399
                    yield _increment_sequence_number_and_return(
1400
                        ResponseTextDeltaEvent(
1401
1402
1403
1404
1405
1406
                            type="response.output_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=delta_message.content,
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
                            logprobs=(
                                self._create_stream_response_logprobs(
                                    token_ids=output.token_ids,
                                    logprobs=output.logprobs,
                                    tokenizer=tokenizer,
                                    top_logprobs=request.top_logprobs,
                                )
                                if request.is_include_output_logprobs()
                                else []
                            ),
1417
1418
                        )
                    )
1419
1420
1421
1422
                current_content_index += 1

                previous_delta_messages.append(delta_message)
        if previous_delta_messages:
1423
            if previous_delta_messages[-1].reasoning is not None:
1424
                reason_content = "".join(
1425
                    pm.reasoning
1426
                    for pm in previous_delta_messages
1427
                    if pm.reasoning is not None
1428
                )
1429
                yield _increment_sequence_number_and_return(
1430
1431
1432
1433
1434
1435
1436
                    ResponseReasoningTextDoneEvent(
                        type="response.reasoning_text.done",
                        item_id=current_item_id,
                        sequence_number=-1,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        text=reason_content,
1437
1438
                    )
                )
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
                current_content_index += 1
                reasoning_item = ResponseReasoningItem(
                    type="reasoning",
                    content=[
                        ResponseReasoningTextContent(
                            text=reason_content,
                            type="reasoning_text",
                        ),
                    ],
                    status="completed",
                    id=current_item_id,
                    summary=[],
                )
1452
                yield _increment_sequence_number_and_return(
1453
1454
1455
1456
1457
                    ResponseOutputItemDoneEvent(
                        type="response.output_item.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        item=reasoning_item,
1458
1459
                    )
                )
1460
            elif previous_delta_messages[-1].content is not None:
1461
1462
1463
1464
1465
                final_content = "".join(
                    pm.content
                    for pm in previous_delta_messages
                    if pm.content is not None
                )
1466
                yield _increment_sequence_number_and_return(
1467
                    ResponseTextDoneEvent(
1468
1469
1470
1471
1472
1473
1474
                        type="response.output_text.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        text=final_content,
                        logprobs=[],
                        item_id=current_item_id,
1475
1476
                    )
                )
1477
1478
1479
1480
1481
1482
                current_content_index += 1
                part = ResponseOutputText(
                    text=final_content,
                    type="output_text",
                    annotations=[],
                )
1483
                yield _increment_sequence_number_and_return(
1484
                    ResponseContentPartDoneEvent(
1485
1486
1487
1488
1489
1490
                        type="response.content_part.done",
                        sequence_number=-1,
                        item_id=current_item_id,
                        output_index=current_output_index,
                        content_index=current_content_index,
                        part=part,
1491
1492
                    )
                )
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
                current_content_index += 1
                item = ResponseOutputMessage(
                    type="message",
                    role="assistant",
                    content=[
                        part,
                    ],
                    status="completed",
                    id=current_item_id,
                    summary=[],
                )
1504
                yield _increment_sequence_number_and_return(
1505
1506
1507
1508
1509
                    ResponseOutputItemDoneEvent(
                        type="response.output_item.done",
                        sequence_number=-1,
                        output_index=current_output_index,
                        item=item,
1510
1511
                    )
                )
1512
1513
1514
1515
1516

    async def _process_harmony_streaming_events(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1517
        result_generator: AsyncIterator[ConversationContext | None],
1518
1519
        context: ConversationContext,
        model_name: str,
1520
        tokenizer: TokenizerLike,
1521
1522
        request_metadata: RequestResponseMetadata,
        created_time: int,
1523
        _increment_sequence_number_and_return: Callable[
1524
1525
            [StreamingResponsesResponse], StreamingResponsesResponse
        ],
1526
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1527
        current_content_index = -1
1528
        current_output_index = 0
1529
        current_item_id: str = ""
1530
        sent_output_item_added = False
1531
        is_first_function_call_delta = False
1532
1533
1534
        async for ctx in result_generator:
            assert isinstance(ctx, StreamingHarmonyContext)

1535
1536
1537
            # finish_reason='error' indicates a retryable error
            self._raise_if_error(ctx.finish_reason, request.request_id)

1538
1539
1540
            if ctx.is_expecting_start():
                current_output_index += 1
                sent_output_item_added = False
1541
                is_first_function_call_delta = False
1542
1543
1544
                if len(ctx.parser.messages) > 0:
                    previous_item = ctx.parser.messages[-1]
                    if previous_item.recipient is not None:
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
                        # Deal with tool call
                        if previous_item.recipient.startswith("functions."):
                            function_name = previous_item.recipient[len("functions.") :]
                            yield _increment_sequence_number_and_return(
                                ResponseFunctionCallArgumentsDoneEvent(
                                    type="response.function_call_arguments.done",
                                    arguments=previous_item.content[0].text,
                                    name=function_name,
                                    item_id=current_item_id,
                                    output_index=current_output_index,
                                    sequence_number=-1,
                                )
                            )
                            function_call_item = ResponseFunctionToolCall(
                                type="function_call",
                                arguments=previous_item.content[0].text,
                                name=function_name,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                sequence_number=-1,
                                call_id=f"fc_{random_uuid()}",
                                status="completed",
                            )
                            yield _increment_sequence_number_and_return(
                                ResponseOutputItemDoneEvent(
                                    type="response.output_item.done",
                                    sequence_number=-1,
                                    output_index=current_output_index,
                                    item=function_call_item,
                                )
                            )
1576
                    elif previous_item.channel == "analysis":
1577
1578
1579
1580
                        content = ResponseReasoningTextContent(
                            text=previous_item.content[0].text,
                            type="reasoning_text",
                        )
1581
1582
                        reasoning_item = ResponseReasoningItem(
                            type="reasoning",
1583
                            content=[content],
1584
                            status="completed",
1585
1586
                            id=current_item_id,
                            summary=[],
1587
                        )
1588
                        yield _increment_sequence_number_and_return(
1589
1590
1591
1592
1593
1594
1595
                            ResponseReasoningTextDoneEvent(
                                type="response.reasoning_text.done",
                                item_id=current_item_id,
                                sequence_number=-1,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                text=previous_item.content[0].text,
1596
1597
                            )
                        )
1598
1599
1600
1601
1602
1603
1604
1605
                        yield _increment_sequence_number_and_return(
                            ResponseReasoningPartDoneEvent(
                                type="response.reasoning_part.done",
                                sequence_number=-1,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                part=content,
1606
1607
                            )
                        )
1608
                        yield _increment_sequence_number_and_return(
1609
1610
1611
1612
1613
                            ResponseOutputItemDoneEvent(
                                type="response.output_item.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item=reasoning_item,
1614
1615
                            )
                        )
1616
1617
1618
1619
1620
1621
                    elif previous_item.channel == "final":
                        text_content = ResponseOutputText(
                            type="output_text",
                            text=previous_item.content[0].text,
                            annotations=[],
                        )
1622
                        yield _increment_sequence_number_and_return(
1623
                            ResponseTextDoneEvent(
1624
1625
1626
1627
1628
1629
1630
                                type="response.output_text.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                text=previous_item.content[0].text,
                                logprobs=[],
                                item_id=current_item_id,
1631
1632
                            )
                        )
1633
                        yield _increment_sequence_number_and_return(
1634
1635
1636
1637
1638
1639
1640
                            ResponseContentPartDoneEvent(
                                type="response.content_part.done",
                                sequence_number=-1,
                                item_id=current_item_id,
                                output_index=current_output_index,
                                content_index=current_content_index,
                                part=text_content,
1641
1642
                            )
                        )
1643
                        yield _increment_sequence_number_and_return(
1644
                            ResponseOutputItemDoneEvent(
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
                                type="response.output_item.done",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item=ResponseOutputMessage(
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[text_content],
                                    status="completed",
                                ),
1655
1656
                            )
                        )
1657

1658
            # stream the output of a harmony message
1659
            if ctx.parser.last_content_delta:
1660
1661
1662
1663
                if (
                    ctx.parser.current_channel == "final"
                    and ctx.parser.current_recipient is None
                ):
1664
1665
                    if not sent_output_item_added:
                        sent_output_item_added = True
1666
                        current_item_id = f"msg_{random_uuid()}"
1667
                        yield _increment_sequence_number_and_return(
1668
1669
1670
1671
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1672
                                item=ResponseOutputMessage(
1673
1674
1675
1676
1677
1678
                                    id=current_item_id,
                                    type="message",
                                    role="assistant",
                                    content=[],
                                    status="in_progress",
                                ),
1679
1680
                            )
                        )
1681
                        current_content_index += 1
1682
                        yield _increment_sequence_number_and_return(
1683
1684
1685
1686
1687
1688
                            ResponseContentPartAddedEvent(
                                type="response.content_part.added",
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
                                content_index=current_content_index,
1689
                                part=ResponseOutputText(
1690
1691
1692
1693
1694
                                    type="output_text",
                                    text="",
                                    annotations=[],
                                    logprobs=[],
                                ),
1695
1696
                            )
                        )
1697
                    yield _increment_sequence_number_and_return(
1698
                        ResponseTextDeltaEvent(
1699
1700
1701
1702
1703
1704
1705
1706
                            type="response.output_text.delta",
                            sequence_number=-1,
                            content_index=current_content_index,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
                            # TODO, use logprobs from ctx.last_request_output
                            logprobs=[],
1707
1708
1709
1710
1711
1712
                        )
                    )
                elif (
                    ctx.parser.current_channel == "analysis"
                    and ctx.parser.current_recipient is None
                ):
1713
1714
                    if not sent_output_item_added:
                        sent_output_item_added = True
1715
                        current_item_id = f"msg_{random_uuid()}"
1716
                        yield _increment_sequence_number_and_return(
1717
1718
1719
1720
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1721
                                item=ResponseReasoningItem(
1722
1723
1724
1725
1726
                                    type="reasoning",
                                    id=current_item_id,
                                    summary=[],
                                    status="in_progress",
                                ),
1727
1728
                            )
                        )
1729
                        current_content_index += 1
1730
                        yield _increment_sequence_number_and_return(
1731
1732
                            ResponseReasoningPartAddedEvent(
                                type="response.reasoning_part.added",
1733
1734
1735
1736
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
                                content_index=current_content_index,
1737
                                part=ResponseReasoningTextContent(
1738
                                    text="",
1739
                                    type="reasoning_text",
1740
                                ),
1741
1742
                            )
                        )
1743
                    yield _increment_sequence_number_and_return(
1744
1745
1746
1747
1748
1749
1750
                        ResponseReasoningTextDeltaEvent(
                            type="response.reasoning_text.delta",
                            item_id=current_item_id,
                            output_index=current_output_index,
                            content_index=current_content_index,
                            delta=ctx.parser.last_content_delta,
                            sequence_number=-1,
1751
1752
                        )
                    )
1753
1754
1755
                # built-in tools will be triggered on the analysis channel
                # However, occasionally built-in tools will
                # still be output to commentary.
1756
1757
1758
1759
                elif (
                    ctx.parser.current_channel == "commentary"
                    or ctx.parser.current_channel == "analysis"
                ) and ctx.parser.current_recipient == "python":
1760
1761
                    if not sent_output_item_added:
                        sent_output_item_added = True
1762
                        current_item_id = f"tool_{random_uuid()}"
1763
                        yield _increment_sequence_number_and_return(
1764
1765
1766
1767
                            ResponseOutputItemAddedEvent(
                                type="response.output_item.added",
                                sequence_number=-1,
                                output_index=current_output_index,
1768
                                item=ResponseCodeInterpreterToolCallParam(
1769
1770
1771
1772
1773
1774
1775
                                    type="code_interpreter_call",
                                    id=current_item_id,
                                    code=None,
                                    container_id="auto",
                                    outputs=None,
                                    status="in_progress",
                                ),
1776
1777
                            )
                        )
1778
                        yield _increment_sequence_number_and_return(
1779
                            ResponseCodeInterpreterCallInProgressEvent(
1780
                                type="response.code_interpreter_call.in_progress",
1781
1782
1783
                                sequence_number=-1,
                                output_index=current_output_index,
                                item_id=current_item_id,
1784
1785
                            )
                        )
1786
                    yield _increment_sequence_number_and_return(
1787
1788
1789
1790
1791
1792
                        ResponseCodeInterpreterCallCodeDeltaEvent(
                            type="response.code_interpreter_call_code.delta",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
1793
1794
                        )
                    )
1795
1796

            # stream tool call outputs
1797
1798
            if ctx.is_assistant_action_turn() and len(ctx.parser.messages) > 0:
                previous_item = ctx.parser.messages[-1]
1799
1800
1801
1802
1803
1804
1805
                if (
                    self.tool_server is not None
                    and self.tool_server.has_tool("browser")
                    and previous_item.recipient is not None
                    and previous_item.recipient.startswith("browser.")
                ):
                    function_name = previous_item.recipient[len("browser.") :]
1806
1807
1808
                    action = None
                    parsed_args = json.loads(previous_item.content[0].text)
                    if function_name == "search":
1809
                        action = response_function_web_search.ActionSearch(
1810
1811
                            type="search",
                            query=parsed_args["query"],
1812
                        )
1813
                    elif function_name == "open":
1814
1815
1816
1817
1818
                        action = response_function_web_search.ActionOpenPage(
                            type="open_page",
                            # TODO: translate to url
                            url=f"cursor:{parsed_args.get('cursor', '')}",
                        )
1819
                    elif function_name == "find":
1820
1821
1822
1823
1824
1825
                        action = response_function_web_search.ActionFind(
                            type="find",
                            pattern=parsed_args["pattern"],
                            # TODO: translate to url
                            url=f"cursor:{parsed_args.get('cursor', '')}",
                        )
1826
                    else:
1827
                        raise ValueError(f"Unknown function name: {function_name}")
1828

1829
                    current_item_id = f"tool_{random_uuid()}"
1830
                    yield _increment_sequence_number_and_return(
1831
                        ResponseOutputItemAddedEvent(
1832
1833
1834
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
1835
                            item=response_function_web_search.ResponseFunctionWebSearch(
1836
1837
1838
1839
1840
1841
                                # TODO: generate a unique id for web search call
                                type="web_search_call",
                                id=current_item_id,
                                action=action,
                                status="in_progress",
                            ),
1842
1843
                        )
                    )
1844
                    yield _increment_sequence_number_and_return(
1845
1846
1847
1848
1849
                        ResponseWebSearchCallInProgressEvent(
                            type="response.web_search_call.in_progress",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1850
1851
                        )
                    )
1852
                    yield _increment_sequence_number_and_return(
1853
1854
1855
1856
1857
                        ResponseWebSearchCallSearchingEvent(
                            type="response.web_search_call.searching",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1858
1859
                        )
                    )
1860
1861

                    # enqueue
1862
                    yield _increment_sequence_number_and_return(
1863
1864
1865
1866
1867
                        ResponseWebSearchCallCompletedEvent(
                            type="response.web_search_call.completed",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1868
1869
                        )
                    )
1870
                    yield _increment_sequence_number_and_return(
1871
                        ResponseOutputItemDoneEvent(
1872
1873
1874
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
1875
                            item=ResponseFunctionWebSearch(
1876
1877
1878
1879
1880
                                type="web_search_call",
                                id=current_item_id,
                                action=action,
                                status="completed",
                            ),
1881
1882
                        )
                    )
1883

1884
1885
1886
1887
1888
1889
                if (
                    self.tool_server is not None
                    and self.tool_server.has_tool("python")
                    and previous_item.recipient is not None
                    and previous_item.recipient.startswith("python")
                ):
1890
                    yield _increment_sequence_number_and_return(
1891
1892
1893
1894
1895
                        ResponseCodeInterpreterCallCodeDoneEvent(
                            type="response.code_interpreter_call_code.done",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1896
                            code=previous_item.content[0].text,
1897
1898
                        )
                    )
1899
                    yield _increment_sequence_number_and_return(
1900
1901
1902
1903
1904
                        ResponseCodeInterpreterCallInterpretingEvent(
                            type="response.code_interpreter_call.interpreting",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1905
1906
                        )
                    )
1907
                    yield _increment_sequence_number_and_return(
1908
1909
1910
1911
1912
                        ResponseCodeInterpreterCallCompletedEvent(
                            type="response.code_interpreter_call.completed",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item_id=current_item_id,
1913
1914
                        )
                    )
1915
                    yield _increment_sequence_number_and_return(
1916
                        ResponseOutputItemDoneEvent(
1917
1918
1919
                            type="response.output_item.done",
                            sequence_number=-1,
                            output_index=current_output_index,
1920
                            item=ResponseCodeInterpreterToolCallParam(
1921
1922
1923
1924
1925
1926
1927
1928
                                type="code_interpreter_call",
                                id=current_item_id,
                                code=previous_item.content[0].text,
                                container_id="auto",
                                # TODO: add outputs here
                                outputs=[],
                                status="completed",
                            ),
1929
1930
                        )
                    )
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
            # developer tools will be triggered on the commentary channel
            # and recipient starts with "functions.TOOL_NAME"
            if (
                ctx.parser.current_channel == "commentary"
                and ctx.parser.current_recipient
                and ctx.parser.current_recipient.startswith("functions.")
            ):
                if is_first_function_call_delta is False:
                    is_first_function_call_delta = True
                    fc_name = ctx.parser.current_recipient[len("functions.") :]
                    tool_call_item = ResponseFunctionToolCall(
                        name=fc_name,
                        type="function_call",
                        id=current_item_id,
                        call_id=f"call_{random_uuid()}",
                        arguments="",
                        status="in_progress",
                    )
                    current_item_id = f"fc_{random_uuid()}"
                    yield _increment_sequence_number_and_return(
                        ResponseOutputItemAddedEvent(
                            type="response.output_item.added",
                            sequence_number=-1,
                            output_index=current_output_index,
                            item=tool_call_item,
                        )
                    )
                else:
                    yield _increment_sequence_number_and_return(
                        ResponseFunctionCallArgumentsDeltaEvent(
                            item_id=current_item_id,
                            delta=ctx.parser.last_content_delta,
                            output_index=current_output_index,
                            sequence_number=-1,
                            type="response.function_call_arguments.delta",
                        )
                    )
1968

1969
1970
1971
1972
    async def responses_stream_generator(
        self,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
1973
        result_generator: AsyncIterator[ConversationContext | None],
1974
1975
        context: ConversationContext,
        model_name: str,
1976
        tokenizer: TokenizerLike,
1977
        request_metadata: RequestResponseMetadata,
1978
        created_time: int | None = None,
1979
    ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1980
1981
1982
1983
1984
        # TODO:
        # 1. Handle disconnect

        created_time = created_time or int(time.time())

1985
1986
        sequence_number = 0

1987
        def _increment_sequence_number_and_return(
1988
            event: StreamingResponsesResponse,
1989
        ) -> StreamingResponsesResponse:
1990
1991
            nonlocal sequence_number
            # Set sequence_number if the event has this attribute
1992
            if hasattr(event, "sequence_number"):
1993
1994
                event.sequence_number = sequence_number
            sequence_number += 1
1995
            return event
1996

1997
        async with AsyncExitStack() as exit_stack:
1998
1999
            processer = None
            if self.use_harmony:
2000
2001
                # TODO: in streaming, we noticed this bug:
                # https://github.com/vllm-project/vllm/issues/25697
2002
                await self._initialize_tool_sessions(request, context, exit_stack)
2003
2004
2005
                processer = self._process_harmony_streaming_events
            else:
                processer = self._process_simple_streaming_events
2006
            # TODO Hanchen make sampling params to include the structural tag
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016

            initial_response = ResponsesResponse.from_request(
                request,
                sampling_params,
                model_name=model_name,
                created_time=created_time,
                output=[],
                status="in_progress",
                usage=None,
            ).model_dump()
2017
            yield _increment_sequence_number_and_return(
2018
2019
2020
2021
                ResponseCreatedEvent(
                    type="response.created",
                    sequence_number=-1,
                    response=initial_response,
2022
2023
                )
            )
2024
            yield _increment_sequence_number_and_return(
2025
2026
2027
2028
                ResponseInProgressEvent(
                    type="response.in_progress",
                    sequence_number=-1,
                    response=initial_response,
2029
2030
                )
            )
2031

2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
            try:
                async for event_data in processer(
                    request,
                    sampling_params,
                    result_generator,
                    context,
                    model_name,
                    tokenizer,
                    request_metadata,
                    created_time,
                    _increment_sequence_number_and_return,
                ):
                    yield event_data
            except GenerationError as e:
                error_json = self._convert_generation_error_to_streaming_response(e)
                yield _increment_sequence_number_and_return(
                    TypeAdapter(StreamingResponsesResponse).validate_json(error_json)
                )
                return
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067

            async def empty_async_generator():
                # A hack to trick Python to think this is a generator but
                # in fact it immediately returns.
                if False:
                    yield

            final_response = await self.responses_full_generator(
                request,
                sampling_params,
                empty_async_generator(),
                context,
                model_name,
                tokenizer,
                request_metadata,
                created_time=created_time,
            )
2068
            yield _increment_sequence_number_and_return(
2069
                ResponseCompletedEvent(
2070
2071
                    type="response.completed",
                    sequence_number=-1,
2072
                    response=final_response,
2073
2074
                )
            )