serving.py 86.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Any, Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from partial_json_parser.core.options import Allow
17

18
from vllm.engine.protocol import EngineClient
19
20
21
22
23
24
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.chat_completion.protocol import (
27
28
29
30
31
32
33
34
35
36
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
37
38
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
39
    TokenState,
40
41
42
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
43
44
45
46
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
47
    FunctionCall,
48
49
50
51
52
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
53
from vllm.entrypoints.openai.engine.serving import (
54
55
56
57
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
58
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
59
60
61
62
63
64
65
66
67
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_inputs_to_harmony_messages,
    parse_chat_output,
    render_for_completion,
)
68
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
69
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
70
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
71
from vllm.logger import init_logger
72
from vllm.logprobs import Logprob
73
from vllm.outputs import CompletionOutput, RequestOutput
74
from vllm.parser import ParserManager
75
from vllm.reasoning import ReasoningParser
76
from vllm.sampling_params import BeamSearchParams, SamplingParams
77
78
79
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
80
81
82
83
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
84
85
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
86
from vllm.tool_parsers.utils import partial_json_loads
87
from vllm.utils.collection_utils import as_list
88
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
89
90
91
92
93

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
94
95
96
    def __init__(
        self,
        engine_client: EngineClient,
97
        models: OpenAIServingModels,
98
99
        response_role: str,
        *,
100
101
        request_logger: RequestLogger | None,
        chat_template: str | None,
102
        chat_template_content_format: ChatTemplateContentFormatOption,
103
        trust_request_chat_template: bool = False,
104
        return_tokens_as_token_ids: bool = False,
105
        reasoning_parser: str = "",
106
        enable_auto_tools: bool = False,
107
        exclude_tools_when_tool_choice_none: bool = False,
108
        tool_parser: str | None = None,
109
        enable_prompt_tokens_details: bool = False,
110
        enable_force_include_usage: bool = False,
111
        enable_log_outputs: bool = False,
112
        enable_log_deltas: bool = True,
113
        log_error_stack: bool = False,
114
        default_chat_template_kwargs: dict[str, Any] | None = None,
115
    ) -> None:
116
117
118
119
120
121
122
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
123

124
        self.response_role = response_role
125
126
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
127
        self.trust_request_chat_template = trust_request_chat_template
128
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
129
        self.enable_log_outputs = enable_log_outputs
130
        self.enable_log_deltas = enable_log_deltas
131

132
133
134
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

135
        # set up reasoning parser
136
        self.reasoning_parser_cls = ParserManager.get_reasoning_parser(
137
138
            reasoning_parser_name=reasoning_parser
        )
139
140
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
141
142
143
144
        self.tool_parser = ParserManager.get_tool_parser(
            tool_parser_name=tool_parser,
            enable_auto_tools=enable_auto_tools,
            model_name=self.model_config.model,
145
146
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
147

148
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
149
        self.enable_force_include_usage = enable_force_include_usage
150
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
151
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
152
153
154
155
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
156
157
                get_stop_tokens_for_assistant_actions()
            )
158

159
160
161
162
163
164
165
166
167
168
        # Handle tool call ID type for Kimi K2 (supporting test mocking via overrides)
        hf_overrides = getattr(self.model_config, "hf_overrides", None)
        if self.model_config.hf_text_config.model_type == "kimi_k2" or (
            isinstance(hf_overrides, dict)
            and hf_overrides.get("model_type") == "kimi_k2"
        ):
            self.tool_call_id_type = "kimi_k2"
        else:
            self.tool_call_id_type = "random"

169
170
171
172
173
174
175
176
177
178
        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                dummy_request.messages,
206
207
208
                default_template=self.chat_template,
                default_template_content_format=self.chat_template_content_format,
                default_template_kwargs=self.default_chat_template_kwargs,
209
210
211
212
213
214
215
216
217
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

218
    async def render_chat_request(
219
220
        self,
        request: ChatCompletionRequest,
221
222
223
224
    ) -> (
        tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]
        | ErrorResponse
    ):
225
        """
226
        render chat request by validating and preprocessing inputs.
227

228
229
230
        Returns:
            A tuple of (conversation, engine_prompts) on success,
            or an ErrorResponse on failure.
231
232
233
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
234
            logger.error("Error with model %s", error_check_ret)
235
236
            return error_check_ret

237
238
239
240
241
242
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

243
        try:
244
245
            renderer = self.engine_client.renderer
            tokenizer = renderer.tokenizer
246

247
248
            tool_parser = self.tool_parser

249
            if isinstance(tokenizer, MistralTokenizer):
250
251
252
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
253
254
                maybe_serialize_tool_calls(request)  # type: ignore[arg-type]
                truncate_tool_call_ids(request)  # type: ignore[arg-type]
255
                validate_request_params(request)
256

257
258
259
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
260
261
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
262
263
264
265
266
267
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
268
            ):
269
270
271
272
273
274
275
276
277
278
279
280
281
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
282

283
284
285
286
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
287
288
289
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
290

291
292
            if not self.use_harmony:
                # Common case.
293
294
295
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
296
                    trust_request_chat_template=self.trust_request_chat_template,
297
298
299
                )
                if error_check_ret is not None:
                    return error_check_ret
300

301
                conversation, engine_prompts = await self._preprocess_chat(
302
303
                    request,
                    request.messages,
304
305
306
                    default_template=self.chat_template,
                    default_template_content_format=self.chat_template_content_format,
                    default_template_kwargs=self.default_chat_template_kwargs,
307
308
309
310
311
                    tool_dicts=tool_dicts,
                    tool_parser=tool_parser,
                )
            else:
                # For GPT-OSS.
312
313
314
315
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
316
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
317
            logger.exception("Error in preprocessing prompt inputs")
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
            return self.create_error_response(e)

        return conversation, engine_prompts

    async def create_chat_completion(
        self,
        request: ChatCompletionRequest,
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
        """
        Chat Completion API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
        Chat Completion API.
        """
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
        # Streaming response
        tokenizer = self.renderer.tokenizer
        assert tokenizer is not None
        reasoning_parser: ReasoningParser | None = None
        try:
            if self.reasoning_parser_cls:
                # Pass the same chat template kwargs as used in tokenization
                chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                    request.chat_template_kwargs,
                    self.default_chat_template_kwargs,
                )
                reasoning_parser = self.reasoning_parser_cls(
                    tokenizer,
                    chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
                )
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            return self.create_error_response(str(e))
352
353
354
355
356
        result = await self.render_chat_request(request)
        if isinstance(result, ErrorResponse):
            return result

        conversation, engine_prompts = result
357

358
359
360
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
361
362
363
364
365

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

366
367
368
369
370
371
372
373
374
375
        try:
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True
            )

            model_name = self.models.model_name(lora_request)
        except (ValueError, TypeError, RuntimeError) as e:
            logger.exception("Error preparing request components")
            return self.create_error_response(e)

376
377
378
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

379
        # Schedule the request and get the result generator.
380
        generators: list[AsyncGenerator[RequestOutput, None]] = []
381
        try:
382
            for i, engine_prompt in enumerate(engine_prompts):
383
                prompt_text = engine_prompt.get("prompt")
384

385
386
387
388
389
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
390
391
392
393

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
394
                    prompt=engine_prompt,
395
396
                    default_sampling_params=self.default_sampling_params,
                )
397

398
                sampling_params: SamplingParams | BeamSearchParams
399
400
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
401
402
                        max_tokens, self.default_sampling_params
                    )
403
404
                else:
                    sampling_params = request.to_sampling_params(
405
406
407
408
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
409
410
411
412
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
413

414
                self._log_inputs(
415
                    sub_request_id,
416
                    engine_prompt,
417
418
419
                    params=sampling_params,
                    lora_request=lora_request,
                )
420

421
422
423
424
425
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
426
427

                if isinstance(sampling_params, BeamSearchParams):
428
                    generator = self.beam_search(
429
                        prompt=engine_prompt,
430
                        request_id=sub_request_id,
431
                        params=sampling_params,
432
                        lora_request=lora_request,
433
                        trace_headers=trace_headers,
434
435
                    )
                else:
436
437
438
439
                    tok_params = request.build_tok_params(self.model_config)
                    tokenization_kwargs = tok_params.get_encode_kwargs()

                    engine_request = self.input_processor.process_inputs(
440
                        sub_request_id,
441
442
443
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
444
                        tokenization_kwargs=tokenization_kwargs,
445
446
                        trace_headers=trace_headers,
                        priority=request.priority,
447
                        data_parallel_rank=data_parallel_rank,
448
                    )
449
450
451
452
453
454
                    reasoning_ended = None
                    if reasoning_parser:
                        reasoning_ended = reasoning_parser.is_reasoning_end(
                            engine_request.prompt_token_ids or []  # type: ignore[attr-defined]
                        )
                        engine_request.reasoning_ended = reasoning_ended
455
                    generator = self.engine_client.generate(
456
                        engine_request,
457
                        sampling_params,
458
                        sub_request_id,
459
460
461
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
462
463
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
464
                        data_parallel_rank=data_parallel_rank,
465
466
467
                    )

                generators.append(generator)
468
        except ValueError as e:
469
            return self.create_error_response(e)
470

471
        assert len(generators) == 1
472
        (result_generator,) = generators
473

474
475
        if request.stream:
            return self.chat_completion_stream_generator(
476
477
478
479
480
481
482
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
483
                reasoning_parser,
484
            )
485

486
487
        try:
            return await self.chat_completion_full_generator(
488
489
490
491
492
493
494
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
495
                reasoning_parser,
496
            )
497
498
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
499
        except ValueError as e:
500
            return self.create_error_response(e)
501
502
503
504

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
505
        return request.messages[-1]["role"]
506

507
    @staticmethod
508
    def _bracket_level(s: str, opening="{", closing="}") -> int:
509
510
511
512
513
514
515
516
517
518
519
520
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
521
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
522
523
524
525
526
527
528
529
530
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
531
            if c == "{":
532
533
                bracket_level += 1
                passed_zero = bracket_level == 0
534
            elif c == "}":
535
536
537
538
539
540
541
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
542
                if c == ",":
543
544
545
546
547
548
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
549
        current_text: str | None,
550
551
        delta_text: str,
        function_name_returned: bool,
552
553
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
554
555
556
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
557
        try:
558
559
560
561
562
563
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
564
            logger.debug("not enough tokens to parse into JSON yet")
565
566
567
568
569
570
571
572
573
574
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
575
576
                delta_text, previous_text
            )
577
578
579
580
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
581
582
583
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
584
585
586
587
588
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
589
590
591
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
592
593
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
594
595
                        arguments, previous_text
                    )
596
597
598
599

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
600
                    if finishes_previous_tool and "parameters" not in current_tool_call:
601
602
603
                        current_tool_call = obj[-2]

                    function_name_returned = True
604
605
606
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
607
608
609
610
611
612
613
614
615
616
617
618
619
620
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
621
622
623

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
624
625
                        delta_text, previous_text
                    )
626
627

                    if delta_text != "":
628
629
630
631
632
633
634
635
636
637
638
639
640
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
641
642
643
644
645
                    else:
                        delta_message = None

        return delta_message, function_name_returned

646
    async def chat_completion_stream_generator(
647
648
649
650
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
651
        model_name: str,
652
        conversation: list[ConversationMessage],
653
        tokenizer: TokenizerLike,
654
        request_metadata: RequestResponseMetadata,
655
        reasoning_parser: ReasoningParser | None = None,
656
    ) -> AsyncGenerator[str, None]:
657
658
        from vllm.tokenizers.mistral import MistralTokenizer

659
        created_time = int(time.time())
660
        chunk_object_type: Final = "chat.completion.chunk"
661
        first_iteration = True
662
663

        # Send response for each token for each request.n (index)
664
665
666
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
667
        num_prompt_tokens = 0
668
        num_cached_tokens = None
669
670
        if self.use_harmony:
            harmony_parsers = [
671
                get_streamable_parser_for_assistant() for _ in range(num_choices)
672
            ]
673
674
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
675
676
677
678
679
680
681
682
683

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
684
685
            and self._should_stream_with_auto_tool_parsing(request)
        )
686

687
        all_previous_token_ids: list[list[int]] | None
688
        function_name_returned = [False] * num_choices
689
        if self.tool_call_id_type == "kimi_k2":
690
691
692
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
693

694
695
696
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

697
698
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
699
        if tool_choice_auto or reasoning_parser:
700
701
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
702
703
704
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
705
            prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
706
        else:
707
            all_previous_token_ids = None
708

709
710
711
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
712
713
714
715
716
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

717
                tool_parsers: list[ToolParser | None] = [
718
719
720
721
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
722
        except Exception as e:
723
            logger.exception("Error in tool parser creation.")
724
            data = self.create_streaming_error_response(e)
725
726
727
728
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

729
        stream_options = request.stream_options
730
731
732
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
733

734
735
        try:
            async for res in result_generator:
736
737
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
738
739
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
740

741
742
743
744
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
745
                    num_cached_tokens = res.num_cached_tokens
746
747
                    # Send first response for each request.n (index) with
                    # the role
748
                    role = self.get_chat_request_role(request)
749
750
751

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
752
                    for i in range(num_choices):
753
754
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
755
756
757
758
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
759
                            logprobs=None,
760
761
                            finish_reason=None,
                        )
762
763

                        # return prompt_token_ids at the first chunk ever
764
765
766
767
768
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
769
                            model=model_name,
770
771
772
773
774
775
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
776

777
778
779
780
781
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
782
783
                                total_tokens=num_prompt_tokens,
                            )
784

785
786
787
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

788
789
                    # Send response to echo the input portion of the
                    # last message
790
                    if request.echo:
791
                        last_msg_content: str | list[dict[str, str]] = ""
792
793
794
795
796
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
797
                            last_msg_content = conversation[-1]["content"] or ""
798
799

                        if last_msg_content:
800
                            for i in range(num_choices):
801
802
803
804
805
806
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
807
808
809
810
811
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
812
813
                                    model=model_name,
                                )
814
815
816
817
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
818
819
                                        total_tokens=num_prompt_tokens,
                                    )
820

821
                                data = chunk.model_dump_json(exclude_unset=True)
822
823
824
825
826
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
827
                    tool_parser = tool_parsers[i]
828

829
                    if (
830
                        reasoning_parser
831
832
833
834
835
836
837
838
                        and res.prompt_token_ids
                        and prompt_is_reasoning_end_arr[i] is None
                    ):
                        # only check once per choice, because prompt_token_ids
                        # are the same for all deltas in that choice
                        prompt_is_reasoning_end_arr[i] = (
                            reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        )
839
840
841
                    if finish_reason_sent[i]:
                        continue

842
                    if request.logprobs and request.top_logprobs is not None:
843
                        assert output.logprobs is not None, "Did not output logprobs"
844
                        logprobs = self._create_chat_logprobs(
845
846
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
847
                            tokenizer=tokenizer,
848
                            num_output_top_logprobs=request.top_logprobs,
849
                            return_as_token_id=request.return_tokens_as_token_ids,
850
851
852
853
                        )
                    else:
                        logprobs = None

854
855
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
856
                        prev_recipient = harmony_parser.current_recipient
857
858
859

                        # Track accumulated content per token with their state
                        token_states: list[TokenState] = []
860
861
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
862
863
864
865
866
867
868
869
870
                            token_delta = harmony_parser.last_content_delta or ""
                            token_states.append(
                                TokenState(
                                    harmony_parser.current_channel,
                                    harmony_parser.current_recipient,
                                    token_delta,
                                )
                            )
                        delta_text = "".join(delta for _, _, delta in token_states)
871
                        cur_channel = harmony_parser.current_channel
872

873
874
875
876
877
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
878
879
                    else:
                        delta_text = output.text
880

881
882
883
884
885
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
886
887
888
                        # Chunked prefill case, don't return empty chunks
                        continue

889
                    delta_message: DeltaMessage | None
890

891
                    # just update previous_texts and previous_token_ids
892
                    if tool_choice_auto or reasoning_parser:
893
894
895
896
897
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
898
899
                        # avoid the None + list error.
                        if previous_token_ids:
900
                            current_token_ids = previous_token_ids + as_list(
901
902
                                output.token_ids
                            )
903
                        else:
904
                            current_token_ids = as_list(output.token_ids)
905

906
                    if self.use_harmony:
907
908
909
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
910
                                token_states=token_states,
911
912
913
914
915
                                prev_recipient=prev_recipient,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
916
                    # handle streaming deltas for tools with named tool_choice
917
                    elif tool_choice_function_name:
918
                        if (
919
                            reasoning_parser
920
921
922
923
924
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
925
926
                            assert reasoning_parser is not None
                            delta_message = (
927
                                reasoning_parser.extract_reasoning_streaming(
928
929
930
931
932
933
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
934
935
                                )
                            )
936
937
938
939
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
940
                            # Only keep 'content', remove 'reasoning'.
941
942
943
                            if (
                                reasoning_parser.is_reasoning_end(
                                    as_list(output.token_ids)
944
                                )
945
                                or prompt_is_reasoning_end_arr[i]
946
                            ):
947
                                reasoning_end_arr[i] = True
948
949
950
951
952
953
954
955
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
956
                            if reasoning_parser:
957
958
959
                                delta_text = previous_text + delta_text
                                current_text = ""

960
961
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
962
963
964
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
965
                            else:
966
967
968
969
970
971
972
973
974
                                # Generate ID based on tokenizer type
                                if isinstance(tokenizer, MistralTokenizer):
                                    tool_call_id = MistralToolCall.generate_random_id()
                                else:
                                    tool_call_id = make_tool_call_id(
                                        id_type=self.tool_call_id_type,
                                        func_name=tool_choice_function_name,
                                        idx=history_tool_call_cnt,
                                    )
975
                                delta_tool_call = DeltaToolCall(
976
                                    id=tool_call_id,
977
978
979
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
980
981
982
983
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
984
                                function_name_returned[i] = True
985
                                history_tool_call_cnt += 1
986

987
988
989
990
991
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
992
                            tools_streamed[i] = True
993

994
995
996
997
998
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
999
1000
1001
                        output_token_ids = as_list(output.token_ids)

                        if (
1002
                            reasoning_parser is not None
1003
                            and not reasoning_end_arr[i]
1004
                            and prompt_is_reasoning_end_arr[i]
1005
1006
                        ):
                            reasoning_end_arr[i] = True
1007

1008
                        if reasoning_parser and not reasoning_end_arr[i]:
1009
                            delta_message = (
1010
                                reasoning_parser.extract_reasoning_streaming(
1011
1012
1013
1014
1015
1016
1017
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
1018
                            )
1019
1020
1021
1022
1023
1024
1025
1026
1027
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

1028
                        else:
1029
                            # either finished reasoning or no reasoning at all
1030
                            content = current_text
1031
1032
1033
1034
1035
1036
1037
1038
1039

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
1040
                            )
1041
1042
1043
1044
1045
1046
1047
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
1048

1049
1050
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
1051
                    elif tool_choice_auto and reasoning_parser:
1052
1053
1054
                        assert tool_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
1055
                        output_token_ids = as_list(output.token_ids)
1056
                        if not reasoning_end_arr[i]:
1057
1058
1059
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
1060
                            if prompt_is_reasoning_end_arr[i]:
1061
                                reasoning_end_arr[i] = True
1062
                                current_token_ids = output_token_ids
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1073
1074
                                    )
                                )
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1092
1093

                        # handle tool calls only after reasoning is done,
1094
                        if reasoning_end_arr[i]:
1095
                            delta_token_ids = output_token_ids
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1106
                            delta_message = tool_parser.extract_tool_calls_streaming(
1107
1108
                                previous_text=previous_text,
                                current_text=current_text,
1109
                                delta_text=delta_text,
1110
1111
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1129
1130
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1131

1132
                    # when only reasoning
1133
                    elif reasoning_parser:
1134
1135
1136
1137
1138
1139
1140
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1141
                        )
1142
                    # handle streaming just a content delta
1143
1144
1145
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1146
                    # update the previous values for the next iteration
1147
                    if (tool_choice_auto or reasoning_parser) and not self.use_harmony:
1148
1149
1150
1151
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1152
1153
1154
1155
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1156

1157
                    # set the previous values for the next iteration
1158
                    previous_num_tokens[i] += len(output.token_ids)
1159
1160
1161
1162
1163
1164

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1165
1166
1167
1168
1169
1170
1171
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1172
                            continue
1173
                        delta_message = DeltaMessage()
1174

1175
1176
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
1177
                        delta_content_parts = []
1178
                        if delta_message.content:
1179
                            delta_content_parts.append(delta_message.content)
1180
1181
                        if delta_message.reasoning:
                            reasoning = delta_message.reasoning
1182
1183
1184
                            delta_content_parts.append(f"[reasoning: {reasoning}]")
                        if delta_message.tool_calls:
                            tool_args = "".join(
1185
1186
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1187
1188
                                if tc.function and tc.function.arguments
                            )
1189
1190
                            if tool_args:
                                delta_content_parts.append(f"[tool_calls: {tool_args}]")
1191

1192
1193
                        if delta_content_parts and self.enable_log_deltas:
                            delta_content = " ".join(delta_content_parts)
1194
1195
1196
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1197
                                output_token_ids=as_list(output.token_ids),
1198
1199
1200
1201
1202
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1203
1204
1205
1206
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1207
                            delta=delta_message,
1208
                            logprobs=logprobs,
1209
                            finish_reason=None,
1210
1211
1212
1213
1214
1215
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1216
1217

                    # if the model is finished generating
1218
                    else:
1219
1220
1221
1222
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1223
1224
1225
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1226
                        # only happens if we are NOT using structured outputs
1227
                        auto_tools_called = False
1228
                        if tool_parser:
1229
1230
1231
1232
1233
1234
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1235
1236
1237
                        else:
                            index = 0

1238
1239
1240
1241
1242
1243
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1244
                            latest_delta_len = 0
1245
1246
                            if (
                                isinstance(
1247
                                    delta_message.tool_calls[0].function,
1248
1249
1250
1251
1252
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1253
                                latest_delta_len = len(
1254
1255
                                    delta_message.tool_calls[0].function.arguments
                                )
1256

1257
1258
1259
1260
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1261
1262
1263
1264
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1265

1266
                            # get what we've streamed so far for arguments
1267
                            # for the current tool
1268
1269
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1270
                                actual_call = actual_call[:-latest_delta_len]
1271
1272

                            # check to see if there's anything left to stream
1273
                            remaining_call = expected_call.replace(actual_call, "", 1)
1274
                            # set that as a delta message
1275
1276
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1277
                            )
1278

1279
                        # Send the finish response for each request.n only once
1280
1281
1282
1283
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1284
1285
                        if (
                            auto_tools_called
1286
                            or (tools_streamed[i] and not tool_choice_function_name)
1287
1288
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1289
1290
                            finish_reason_ = "tool_calls"
                        else:
1291
1292
1293
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1294
1295
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1296
                            delta=delta_message,
1297
                            logprobs=logprobs,
1298
                            finish_reason=finish_reason_,
1299
                            stop_reason=output.stop_reason,
1300
1301
1302
1303
1304
1305
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1306

1307
                        finish_reason_sent[i] = True
1308

1309
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1310
1311
1312
1313
1314
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1315
1316
                        model=model_name,
                    )
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1327
                    data = chunk.model_dump_json(exclude_unset=True)
1328
1329
                    yield f"data: {data}\n\n"

1330
1331
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1332
1333
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1334
1335
1336
1337
1338
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1339
1340
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1341
1342
                        cached_tokens=num_cached_tokens
                    )
1343
1344
1345
1346
1347
1348
1349

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1350
1351
1352
1353
1354
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1355
                yield f"data: {final_usage_data}\n\n"
1356

1357
1358
1359
1360
1361
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1362
1363
1364
1365
1366
1367
1368
1369
1370
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1371
1372
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1373
1374
1375
1376
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1377
                        output_token_ids=None,  # Consider also logging all token IDs
1378
1379
1380
1381
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1382

1383
1384
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1385
        except Exception as e:
1386
            logger.exception("Error in chat completion stream generator.")
1387
            data = self.create_streaming_error_response(e)
1388
            yield f"data: {data}\n\n"
1389
1390
1391
1392
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1393
1394
1395
1396
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1397
        model_name: str,
1398
        conversation: list[ConversationMessage],
1399
        tokenizer: TokenizerLike,
1400
        request_metadata: RequestResponseMetadata,
1401
        reasoning_parser: ReasoningParser | None = None,
1402
    ) -> ErrorResponse | ChatCompletionResponse:
1403
1404
        from vllm.tokenizers.mistral import MistralTokenizer

1405
        created_time = int(time.time())
1406
        final_res: RequestOutput | None = None
1407

1408
1409
1410
1411
1412
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1413
        except ValueError as e:
1414
            return self.create_error_response(e)
1415

1416
1417
        assert final_res is not None

1418
        choices: list[ChatCompletionResponseChoice] = []
1419
        if self.tool_call_id_type == "kimi_k2":
1420
1421
1422
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1423

1424
1425
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1426
1427
1428
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1429
            token_ids = output.token_ids
1430
            out_logprobs = output.logprobs
1431
            tool_call_info = None
1432

1433
1434
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1435
                logprobs = self._create_chat_logprobs(
1436
                    token_ids=token_ids,
1437
                    top_logprobs=out_logprobs,
1438
                    num_output_top_logprobs=request.top_logprobs,
1439
                    tokenizer=tokenizer,
1440
                    return_as_token_id=request.return_tokens_as_token_ids,
1441
1442
1443
                )
            else:
                logprobs = None
1444
1445

            if self.use_harmony:
1446
                reasoning, content, _ = parse_chat_output(token_ids)
1447
                if not request.include_reasoning:
1448
                    reasoning = None
1449

1450
                if self.tool_parser is not None:
1451
1452
1453
1454
1455
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1456
1457
1458
1459
1460
1461
1462
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1463
                    content = tool_call_info.content
1464
1465
                    message = ChatMessage(
                        role=role,
1466
                        reasoning=reasoning,
1467
1468
1469
1470
1471
1472
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1473
                        reasoning=reasoning,
1474
1475
                        content=content,
                    )
1476
1477
1478
1479
1480

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1481
1482
1483
1484
1485
1486
1487
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1488
                    stop_reason=output.stop_reason,
1489
1490
1491
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1492
1493
1494
                )
                choices.append(choice_data)
                continue
1495

1496
            if reasoning_parser:
1497
1498
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1499
                reasoning, content = reasoning_parser.extract_reasoning(
1500
1501
                    output.text, request=request
                )
1502
                if not request.include_reasoning:
1503
                    reasoning = None
1504
            else:
1505
                reasoning = None
1506
                content = output.text
1507

1508
            auto_tools_called = False
1509
1510
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
            if self.use_harmony:
                # Harmony models already have parsed content and tool_calls
                # through parse_chat_output. Respect its output directly.
                message = ChatMessage(
                    role=role,
                    reasoning=reasoning,
                    content=content,
                    tool_calls=tool_calls if tool_calls else [],
                )

            elif (not self.enable_auto_tools or not self.tool_parser) and (
1532
1533
1534
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1535
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1536

1537
1538
1539
1540
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1541
                assert tool_calls is not None and len(tool_calls) > 0
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
                tool_call_class_items = []
                for idx, tc in enumerate(tool_calls):
                    # Use native ID if available (e.g., Kimi K2),
                    # otherwise generate ID with correct id_type
                    if tc.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tc.id, function=tc)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(tool_call_class(function=tc))
                        else:
                            generated_id = make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tc.name,
1560
                                idx=history_tool_call_cnt,
1561
1562
1563
1564
1565
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tc)
                            )
                    history_tool_call_cnt += 1
1566
1567
                message = ChatMessage(
                    role=role,
1568
                    reasoning=reasoning,
1569
                    content="",
1570
                    tool_calls=tool_call_class_items,
1571
                )
1572

1573
            elif request.tool_choice and request.tool_choice == "required":
1574
1575
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
                for idx, tool_call in enumerate(tool_calls):
                    # Use native ID if available,
                    # otherwise generate ID with correct id_type
                    if tool_call.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tool_call.id, function=tool_call)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(
                                tool_call_class(function=tool_call)
                            )
                        else:
                            generated_id = make_tool_call_id(
1593
1594
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
1595
                                idx=history_tool_call_cnt,
1596
1597
1598
1599
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tool_call)
                            )
1600
                    history_tool_call_cnt += 1
1601
1602
1603
                message = ChatMessage(
                    role=role,
                    content="",
1604
                    tool_calls=tool_call_class_items,
1605
                    reasoning=reasoning,
1606
                )
1607

1608
1609
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1610
            elif not request.tool_choice or request.tool_choice == "none":
1611
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1612
1613

            # handle when there are tools and tool choice is auto
1614
1615
1616
1617
1618
1619
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1620
1621
1622
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1623
1624
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
                    tool_call_items = []
                    for idx, tc in enumerate(tool_calls):
                        # Use native ID if available (e.g., Kimi K2),
                        # otherwise generate ID with correct id_type
                        if tc.id:
                            tool_call_items.append(
                                tool_call_class(id=tc.id, function=tc)
                            )
                        else:
                            # Generate ID using the correct format (kimi_k2 or random),
                            # but leave it to the class if it's Mistral to preserve
                            # 9-char IDs
                            if isinstance(tokenizer, MistralTokenizer):
                                tool_call_items.append(tool_call_class(function=tc))
                            else:
                                generated_id = make_tool_call_id(
                                    id_type=self.tool_call_id_type,
                                    func_name=tc.name,
1643
                                    idx=history_tool_call_cnt,
1644
1645
1646
1647
1648
                                )
                                tool_call_items.append(
                                    tool_call_class(id=generated_id, function=tc)
                                )
                        history_tool_call_cnt += 1
1649
1650
                    message = ChatMessage(
                        role=role,
1651
                        reasoning=reasoning,
1652
                        content=content,
1653
                        tool_calls=tool_call_items,
1654
                    )
1655
1656
1657
1658

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1659
1660
1661
1662
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1663
1664
                    if content and len(content) > 0:
                        ret_content = content
1665
1666
                    message = ChatMessage(
                        role=role,
1667
                        reasoning=reasoning,
1668
1669
                        content=ret_content,
                    )
1670
1671
1672
1673
1674
1675

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1676
1677
                    "completion."
                )
1678
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1679
1680
1681
1682
1683
1684
1685
1686
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1687

1688
1689
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1690
                message=message,
1691
                logprobs=logprobs,
1692
1693
1694
1695
1696
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1697
                stop_reason=output.stop_reason,
1698
1699
1700
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1701
            )
1702
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1703

1704
1705
            choices.append(choice_data)

1706
        if request.echo:
1707
            last_msg_content: str | list[dict[str, str]] = ""
1708
1709
1710
1711
1712
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1713
                last_msg_content = conversation[-1]["content"] or ""
1714
            if isinstance(last_msg_content, list):
1715
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1716
1717

            for choice in choices:
1718
                full_message = last_msg_content + (choice.message.content or "")
1719
1720
                choice.message.content = full_message

1721
        assert final_res.prompt_token_ids is not None
1722
        num_prompt_tokens = len(final_res.prompt_token_ids)
1723
1724
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1725
        num_generated_tokens = sum(
1726
1727
1728
1729
1730
1731
1732
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1733
1734
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1735
1736
                cached_tokens=final_res.num_cached_tokens
            )
1737
1738
1739

        request_metadata.final_usage_info = usage

1740
1741
1742
1743
1744
1745
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1746
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1747
1748
1749
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1750
            kv_transfer_params=final_res.kv_transfer_params,
1751
1752
        )

1753
1754
1755
1756
1757
1758
1759
1760
1761
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1762
1763
1764
1765
1766
                    for tc in choice.message.tool_calls:  # type: ignore
                        function_call: FunctionCall = tc.function  # type: ignore
                        tool_call_descriptions.append(
                            f"{function_call.name}({function_call.arguments})"
                        )
1767
1768
1769
1770
1771
1772
1773
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1774
                        output_token_ids = final_res.outputs[choice.index].token_ids
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1785
        return response
1786
1787

    def _get_top_logprobs(
1788
1789
        self,
        logprobs: dict[int, Logprob],
1790
        top_logprobs: int | None,
1791
        tokenizer: TokenizerLike | None,
1792
1793
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1794
        return [
1795
            ChatCompletionLogProb(
1796
1797
1798
1799
1800
1801
1802
1803
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1804
1805
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1806
1807
            )
            for i, p in enumerate(logprobs.items())
1808
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1809
1810
1811
1812
1813
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1814
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1815
        tokenizer: TokenizerLike | None,
1816
1817
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1818
1819
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1820
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1821

1822
1823
1824
1825
1826
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1827
1828
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1829
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1830
                if should_return_as_token_id:
1831
                    token = f"token_id:{token_id}"
1832
                else:
1833
1834
                    if tokenizer is None:
                        raise ValueError(
1835
                            "Unable to get tokenizer because `skip_tokenizer_init=True`"
1836
1837
                        )

1838
                    token = tokenizer.decode(token_id)
1839

1840
1841
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1842
                        token=token,
1843
                        bytes=list(token.encode("utf-8", errors="replace")),
1844
1845
                    )
                )
1846
            else:
1847
1848
1849
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1850
1851
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1852
                        token=self._get_decoded_token(
1853
1854
1855
                            step_token,
                            token_id,
                            tokenizer,
1856
                            should_return_as_token_id,
1857
1858
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1859
1860
1861
1862
1863
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1864
                        top_logprobs=self._get_top_logprobs(
1865
1866
1867
1868
1869
1870
1871
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1872
1873

        return ChatCompletionLogProbs(content=logprobs_content)
1874

1875
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1876
1877
1878
1879
1880
1881
1882
1883
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1884
1885
1886
1887
1888
1889
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1890
1891
1892

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1893
        delta_message: DeltaMessage | None,
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1905
            output.finish_reason is not None
1906
1907
1908
1909
1910
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1911
1912
1913
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1914

1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )

1944
1945
1946
    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1947
        should_include_tools: bool = True,
1948
1949
1950
    ):
        messages: list[OpenAIMessage] = []

1951
1952
1953
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
1954
        maybe_serialize_tool_calls(request)  # type: ignore[arg-type]
1955

1956
1957
1958
1959
1960
1961
1962
1963
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1964
            python_description=None,
1965
            with_custom_tools=should_include_tools,
1966
        )
1967
1968
1969
        messages.append(sys_msg)

        # Add developer message.
1970
1971
        if request.tools:
            dev_msg = get_developer_message(
1972
                tools=request.tools if should_include_tools else None  # type: ignore[arg-type]
1973
1974
            )
            messages.append(dev_msg)
1975
1976

        # Add user message.
1977
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1978
1979
1980

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1981
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1982
1983
1984
1985
1986

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1987
        return messages, [engine_prompt]