serving.py 87.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Any, Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from partial_json_parser.core.options import Allow
17

18
from vllm.engine.protocol import EngineClient
19
20
21
22
23
24
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.chat_completion.protocol import (
27
28
29
30
31
32
33
34
35
36
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
37
38
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
39
    TokenState,
40
41
42
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
43
44
45
46
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
47
    FunctionCall,
48
49
50
51
52
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
53
from vllm.entrypoints.openai.engine.serving import (
54
55
56
57
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
58
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
59
60
61
62
63
64
65
66
67
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_inputs_to_harmony_messages,
    parse_chat_output,
    render_for_completion,
)
68
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
69
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
70
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
71
from vllm.logger import init_logger
72
from vllm.logprobs import Logprob
73
from vllm.outputs import CompletionOutput, RequestOutput
74
from vllm.parser import ParserManager
75
from vllm.sampling_params import BeamSearchParams, SamplingParams
76
77
78
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
79
80
81
82
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
83
84
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
85
from vllm.tool_parsers.utils import partial_json_loads
86
from vllm.utils.collection_utils import as_list
87
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
88
89
90
91
92

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
93
94
95
    def __init__(
        self,
        engine_client: EngineClient,
96
        models: OpenAIServingModels,
97
98
        response_role: str,
        *,
99
100
        request_logger: RequestLogger | None,
        chat_template: str | None,
101
        chat_template_content_format: ChatTemplateContentFormatOption,
102
        trust_request_chat_template: bool = False,
103
        return_tokens_as_token_ids: bool = False,
104
        reasoning_parser: str = "",
105
        enable_auto_tools: bool = False,
106
        exclude_tools_when_tool_choice_none: bool = False,
107
        tool_parser: str | None = None,
108
        enable_prompt_tokens_details: bool = False,
109
        enable_force_include_usage: bool = False,
110
        enable_log_outputs: bool = False,
111
        enable_log_deltas: bool = True,
112
        log_error_stack: bool = False,
113
        default_chat_template_kwargs: dict[str, Any] | None = None,
114
    ) -> None:
115
116
117
118
119
120
121
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
122

123
        self.response_role = response_role
124
125
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
126
        self.trust_request_chat_template = trust_request_chat_template
127
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
128
        self.enable_log_outputs = enable_log_outputs
129
        self.enable_log_deltas = enable_log_deltas
130

131
132
133
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

134
        # set up reasoning parser
135
        self.reasoning_parser = ParserManager.get_reasoning_parser(
136
137
            reasoning_parser_name=reasoning_parser
        )
138
139
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
140
141
142
143
        self.tool_parser = ParserManager.get_tool_parser(
            tool_parser_name=tool_parser,
            enable_auto_tools=enable_auto_tools,
            model_name=self.model_config.model,
144
145
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
146

147
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
148
        self.enable_force_include_usage = enable_force_include_usage
149
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
150
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
151
152
153
154
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
155
156
                get_stop_tokens_for_assistant_actions()
            )
157

158
159
160
161
162
163
164
165
166
167
        # Handle tool call ID type for Kimi K2 (supporting test mocking via overrides)
        hf_overrides = getattr(self.model_config, "hf_overrides", None)
        if self.model_config.hf_text_config.model_type == "kimi_k2" or (
            isinstance(hf_overrides, dict)
            and hf_overrides.get("model_type") == "kimi_k2"
        ):
            self.tool_call_id_type = "kimi_k2"
        else:
            self.tool_call_id_type = "random"

168
169
170
171
172
173
174
175
176
177
        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                dummy_request.messages,
205
206
207
                default_template=self.chat_template,
                default_template_content_format=self.chat_template_content_format,
                default_template_kwargs=self.default_chat_template_kwargs,
208
209
210
211
212
213
214
215
216
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

217
    async def render_chat_request(
218
219
        self,
        request: ChatCompletionRequest,
220
221
222
223
    ) -> (
        tuple[list[ConversationMessage], list[TokensPrompt | EmbedsPrompt]]
        | ErrorResponse
    ):
224
        """
225
        render chat request by validating and preprocessing inputs.
226

227
228
229
        Returns:
            A tuple of (conversation, engine_prompts) on success,
            or an ErrorResponse on failure.
230
231
232
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
233
            logger.error("Error with model %s", error_check_ret)
234
235
            return error_check_ret

236
237
238
239
240
241
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

242
        try:
243
244
            renderer = self.engine_client.renderer
            tokenizer = renderer.tokenizer
245

246
247
            tool_parser = self.tool_parser

248
            if isinstance(tokenizer, MistralTokenizer):
249
250
251
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
252
253
                maybe_serialize_tool_calls(request)  # type: ignore[arg-type]
                truncate_tool_call_ids(request)  # type: ignore[arg-type]
254
                validate_request_params(request)
255

256
257
258
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
259
260
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
261
262
263
264
265
266
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
267
            ):
268
269
270
271
272
273
274
275
276
277
278
279
280
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
281

282
283
284
285
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
286
287
288
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
289

290
291
            if not self.use_harmony:
                # Common case.
292
293
294
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
295
                    trust_request_chat_template=self.trust_request_chat_template,
296
297
298
                )
                if error_check_ret is not None:
                    return error_check_ret
299

300
                conversation, engine_prompts = await self._preprocess_chat(
301
302
                    request,
                    request.messages,
303
304
305
                    default_template=self.chat_template,
                    default_template_content_format=self.chat_template_content_format,
                    default_template_kwargs=self.default_chat_template_kwargs,
306
307
308
309
310
                    tool_dicts=tool_dicts,
                    tool_parser=tool_parser,
                )
            else:
                # For GPT-OSS.
311
312
313
314
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
315
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
316
            logger.exception("Error in preprocessing prompt inputs")
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
            return self.create_error_response(e)

        return conversation, engine_prompts

    async def create_chat_completion(
        self,
        request: ChatCompletionRequest,
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
        """
        Chat Completion API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
        Chat Completion API.
        """
        result = await self.render_chat_request(request)
        if isinstance(result, ErrorResponse):
            return result

        conversation, engine_prompts = result
338

339
340
341
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
342
343
344
345
346

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

347
348
349
350
351
352
353
354
355
356
        try:
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True
            )

            model_name = self.models.model_name(lora_request)
        except (ValueError, TypeError, RuntimeError) as e:
            logger.exception("Error preparing request components")
            return self.create_error_response(e)

357
358
359
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

360
        # Schedule the request and get the result generator.
361
        generators: list[AsyncGenerator[RequestOutput, None]] = []
362
        try:
363
            for i, engine_prompt in enumerate(engine_prompts):
364
                prompt_text = engine_prompt.get("prompt")
365

366
367
368
369
370
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
371
372
373
374

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
375
                    prompt=engine_prompt,
376
377
                    default_sampling_params=self.default_sampling_params,
                )
378

379
                sampling_params: SamplingParams | BeamSearchParams
380
381
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
382
383
                        max_tokens, self.default_sampling_params
                    )
384
385
                else:
                    sampling_params = request.to_sampling_params(
386
387
388
389
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
390
391
392
393
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
394

395
                self._log_inputs(
396
                    sub_request_id,
397
                    engine_prompt,
398
399
400
                    params=sampling_params,
                    lora_request=lora_request,
                )
401

402
403
404
405
406
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
407
408

                if isinstance(sampling_params, BeamSearchParams):
409
                    generator = self.beam_search(
410
                        prompt=engine_prompt,
411
                        request_id=sub_request_id,
412
                        params=sampling_params,
413
                        lora_request=lora_request,
414
                        trace_headers=trace_headers,
415
416
                    )
                else:
417
418
419
420
                    tok_params = request.build_tok_params(self.model_config)
                    tokenization_kwargs = tok_params.get_encode_kwargs()

                    engine_request = self.input_processor.process_inputs(
421
                        sub_request_id,
422
423
424
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
425
                        tokenization_kwargs=tokenization_kwargs,
426
427
                        trace_headers=trace_headers,
                        priority=request.priority,
428
                        data_parallel_rank=data_parallel_rank,
429
                    )
430

431
                    generator = self.engine_client.generate(
432
                        engine_request,
433
                        sampling_params,
434
                        sub_request_id,
435
436
437
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
438
439
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
440
                        data_parallel_rank=data_parallel_rank,
441
442
443
                    )

                generators.append(generator)
444
        except ValueError as e:
445
            return self.create_error_response(e)
446

447
        assert len(generators) == 1
448
        (result_generator,) = generators
449

450
        # Streaming response
451
        tokenizer = self.renderer.tokenizer
452
        assert tokenizer is not None
453

454
455
        if request.stream:
            return self.chat_completion_stream_generator(
456
457
458
459
460
461
462
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
463
            )
464

465
466
        try:
            return await self.chat_completion_full_generator(
467
468
469
470
471
472
473
474
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
475
476
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
477
        except ValueError as e:
478
            return self.create_error_response(e)
479
480
481
482

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
483
        return request.messages[-1]["role"]
484

485
    @staticmethod
486
    def _bracket_level(s: str, opening="{", closing="}") -> int:
487
488
489
490
491
492
493
494
495
496
497
498
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
499
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
500
501
502
503
504
505
506
507
508
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
509
            if c == "{":
510
511
                bracket_level += 1
                passed_zero = bracket_level == 0
512
            elif c == "}":
513
514
515
516
517
518
519
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
520
                if c == ",":
521
522
523
524
525
526
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
527
        current_text: str | None,
528
529
        delta_text: str,
        function_name_returned: bool,
530
531
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
532
533
534
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
535
        try:
536
537
538
539
540
541
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
542
            logger.debug("not enough tokens to parse into JSON yet")
543
544
545
546
547
548
549
550
551
552
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
553
554
                delta_text, previous_text
            )
555
556
557
558
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
559
560
561
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
562
563
564
565
566
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
567
568
569
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
570
571
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
572
573
                        arguments, previous_text
                    )
574
575
576
577

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
578
                    if finishes_previous_tool and "parameters" not in current_tool_call:
579
580
581
                        current_tool_call = obj[-2]

                    function_name_returned = True
582
583
584
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
585
586
587
588
589
590
591
592
593
594
595
596
597
598
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
599
600
601

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
602
603
                        delta_text, previous_text
                    )
604
605

                    if delta_text != "":
606
607
608
609
610
611
612
613
614
615
616
617
618
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
619
620
621
622
623
                    else:
                        delta_message = None

        return delta_message, function_name_returned

624
    async def chat_completion_stream_generator(
625
626
627
628
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
629
        model_name: str,
630
        conversation: list[ConversationMessage],
631
        tokenizer: TokenizerLike,
632
        request_metadata: RequestResponseMetadata,
633
    ) -> AsyncGenerator[str, None]:
634
635
        from vllm.tokenizers.mistral import MistralTokenizer

636
        created_time = int(time.time())
637
        chunk_object_type: Final = "chat.completion.chunk"
638
        first_iteration = True
639
640

        # Send response for each token for each request.n (index)
641
642
643
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
644
        num_prompt_tokens = 0
645
        num_cached_tokens = None
646
647
        if self.use_harmony:
            harmony_parsers = [
648
                get_streamable_parser_for_assistant() for _ in range(num_choices)
649
            ]
650
651
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
652
653
654
655
656
657
658
659
660

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
661
662
            and self._should_stream_with_auto_tool_parsing(request)
        )
663

664
        all_previous_token_ids: list[list[int]] | None
665
        function_name_returned = [False] * num_choices
666
        if self.tool_call_id_type == "kimi_k2":
667
668
669
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
670

671
672
673
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

674
675
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
676
        if tool_choice_auto or self.reasoning_parser:
677
678
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
679
680
681
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
682
            prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
683
        else:
684
            all_previous_token_ids = None
685

686
        try:
687
            if self.reasoning_parser:
688
689
690
691
692
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

693
694
695
696
697
                # Pass the same chat template kwargs as used in tokenization
                chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                    request.chat_template_kwargs,
                    self.default_chat_template_kwargs,
                )
698
699
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
700
                    chat_template_kwargs=chat_template_kwargs or {},  # type: ignore[call-arg]
701
                )
702
703
704
705
706
707
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
708
709
710
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
711
712
713
714
715
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

716
                tool_parsers: list[ToolParser | None] = [
717
718
719
720
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
721
        except Exception as e:
722
            logger.exception("Error in tool parser creation.")
723
            data = self.create_streaming_error_response(e)
724
725
726
727
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

728
        stream_options = request.stream_options
729
730
731
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
732

733
734
        try:
            async for res in result_generator:
735
736
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
737
738
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
739

740
741
742
743
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
744
                    num_cached_tokens = res.num_cached_tokens
745
746
                    # Send first response for each request.n (index) with
                    # the role
747
                    role = self.get_chat_request_role(request)
748
749
750

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
751
                    for i in range(num_choices):
752
753
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
754
755
756
757
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
758
                            logprobs=None,
759
760
                            finish_reason=None,
                        )
761
762

                        # return prompt_token_ids at the first chunk ever
763
764
765
766
767
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
768
                            model=model_name,
769
770
771
772
773
774
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
775

776
777
778
779
780
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
781
782
                                total_tokens=num_prompt_tokens,
                            )
783

784
785
786
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

787
788
                    # Send response to echo the input portion of the
                    # last message
789
                    if request.echo:
790
                        last_msg_content: str | list[dict[str, str]] = ""
791
792
793
794
795
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
796
                            last_msg_content = conversation[-1]["content"] or ""
797
798

                        if last_msg_content:
799
                            for i in range(num_choices):
800
801
802
803
804
805
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
806
807
808
809
810
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
811
812
                                    model=model_name,
                                )
813
814
815
816
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
817
818
                                        total_tokens=num_prompt_tokens,
                                    )
819

820
                                data = chunk.model_dump_json(exclude_unset=True)
821
822
823
824
825
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
826
                    tool_parser = tool_parsers[i]
827

828
829
830
831
832
833
834
835
836
837
                    if (
                        self.reasoning_parser
                        and res.prompt_token_ids
                        and prompt_is_reasoning_end_arr[i] is None
                    ):
                        # only check once per choice, because prompt_token_ids
                        # are the same for all deltas in that choice
                        prompt_is_reasoning_end_arr[i] = (
                            reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        )
838
839
840
                    if finish_reason_sent[i]:
                        continue

841
                    if request.logprobs and request.top_logprobs is not None:
842
                        assert output.logprobs is not None, "Did not output logprobs"
843
                        logprobs = self._create_chat_logprobs(
844
845
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
846
                            tokenizer=tokenizer,
847
                            num_output_top_logprobs=request.top_logprobs,
848
                            return_as_token_id=request.return_tokens_as_token_ids,
849
850
851
852
                        )
                    else:
                        logprobs = None

853
854
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
855
                        prev_recipient = harmony_parser.current_recipient
856
857
858

                        # Track accumulated content per token with their state
                        token_states: list[TokenState] = []
859
860
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
861
862
863
864
865
866
867
868
869
                            token_delta = harmony_parser.last_content_delta or ""
                            token_states.append(
                                TokenState(
                                    harmony_parser.current_channel,
                                    harmony_parser.current_recipient,
                                    token_delta,
                                )
                            )
                        delta_text = "".join(delta for _, _, delta in token_states)
870
                        cur_channel = harmony_parser.current_channel
871

872
873
874
875
876
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
877
878
                    else:
                        delta_text = output.text
879

880
881
882
883
884
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
885
886
887
                        # Chunked prefill case, don't return empty chunks
                        continue

888
                    delta_message: DeltaMessage | None
889

890
                    # just update previous_texts and previous_token_ids
891
                    if tool_choice_auto or self.reasoning_parser:
892
893
894
895
896
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
897
898
                        # avoid the None + list error.
                        if previous_token_ids:
899
                            current_token_ids = previous_token_ids + as_list(
900
901
                                output.token_ids
                            )
902
                        else:
903
                            current_token_ids = as_list(output.token_ids)
904

905
                    if self.use_harmony:
906
907
908
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
909
                                token_states=token_states,
910
911
912
913
914
                                prev_recipient=prev_recipient,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
915
                    # handle streaming deltas for tools with named tool_choice
916
                    elif tool_choice_function_name:
917
918
919
920
921
922
923
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
924
925
                            assert reasoning_parser is not None
                            delta_message = (
926
                                reasoning_parser.extract_reasoning_streaming(
927
928
929
930
931
932
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
933
934
                                )
                            )
935
936
937
938
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
939
                            # Only keep 'content', remove 'reasoning'.
940
941
942
                            if (
                                reasoning_parser.is_reasoning_end(
                                    as_list(output.token_ids)
943
                                )
944
                                or prompt_is_reasoning_end_arr[i]
945
                            ):
946
                                reasoning_end_arr[i] = True
947
948
949
950
951
952
953
954
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
955
                            if self.reasoning_parser:
956
957
958
                                delta_text = previous_text + delta_text
                                current_text = ""

959
960
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
961
962
963
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
964
                            else:
965
966
967
968
969
970
971
972
973
                                # Generate ID based on tokenizer type
                                if isinstance(tokenizer, MistralTokenizer):
                                    tool_call_id = MistralToolCall.generate_random_id()
                                else:
                                    tool_call_id = make_tool_call_id(
                                        id_type=self.tool_call_id_type,
                                        func_name=tool_choice_function_name,
                                        idx=history_tool_call_cnt,
                                    )
974
                                delta_tool_call = DeltaToolCall(
975
                                    id=tool_call_id,
976
977
978
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
979
980
981
982
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
983
                                function_name_returned[i] = True
984
                                history_tool_call_cnt += 1
985

986
987
988
989
990
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
991
                            tools_streamed[i] = True
992

993
994
995
996
997
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
998
999
1000
1001
1002
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
1003
                            and prompt_is_reasoning_end_arr[i]
1004
1005
                        ):
                            reasoning_end_arr[i] = True
1006

1007
1008
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
1009
                                reasoning_parser.extract_reasoning_streaming(
1010
1011
1012
1013
1014
1015
1016
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
1017
                            )
1018
1019
1020
1021
1022
1023
1024
1025
1026
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

1027
                        else:
1028
                            # either finished reasoning or no reasoning at all
1029
                            content = current_text
1030
1031
1032
1033
1034
1035
1036
1037
1038

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
1039
                            )
1040
1041
1042
1043
1044
1045
1046
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
1047

1048
1049
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
1050
                    elif tool_choice_auto and self.reasoning_parser:
1051
1052
1053
1054
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
1055
                        output_token_ids = as_list(output.token_ids)
1056
                        if not reasoning_end_arr[i]:
1057
1058
1059
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
1060
                            if prompt_is_reasoning_end_arr[i]:
1061
                                reasoning_end_arr[i] = True
1062
                                current_token_ids = output_token_ids
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1073
1074
                                    )
                                )
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1092
1093

                        # handle tool calls only after reasoning is done,
1094
                        if reasoning_end_arr[i]:
1095
                            delta_token_ids = output_token_ids
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1106
                            delta_message = tool_parser.extract_tool_calls_streaming(
1107
1108
                                previous_text=previous_text,
                                current_text=current_text,
1109
                                delta_text=delta_text,
1110
1111
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1129
1130
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1131

1132
                    # when only reasoning
1133
                    elif self.reasoning_parser:
1134
1135
1136
1137
1138
1139
1140
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1141
                        )
1142
                    # handle streaming just a content delta
1143
1144
1145
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1146
                    # update the previous values for the next iteration
1147
1148
1149
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1150
1151
1152
1153
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1154
1155
1156
1157
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1158

1159
                    # set the previous values for the next iteration
1160
                    previous_num_tokens[i] += len(output.token_ids)
1161
1162
1163
1164
1165
1166

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1167
1168
1169
1170
1171
1172
1173
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1174
                            continue
1175
                        delta_message = DeltaMessage()
1176

1177
1178
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
1179
                        delta_content_parts = []
1180
                        if delta_message.content:
1181
                            delta_content_parts.append(delta_message.content)
1182
1183
                        if delta_message.reasoning:
                            reasoning = delta_message.reasoning
1184
1185
1186
                            delta_content_parts.append(f"[reasoning: {reasoning}]")
                        if delta_message.tool_calls:
                            tool_args = "".join(
1187
1188
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1189
1190
                                if tc.function and tc.function.arguments
                            )
1191
1192
                            if tool_args:
                                delta_content_parts.append(f"[tool_calls: {tool_args}]")
1193

1194
1195
                        if delta_content_parts and self.enable_log_deltas:
                            delta_content = " ".join(delta_content_parts)
1196
1197
1198
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1199
                                output_token_ids=as_list(output.token_ids),
1200
1201
1202
1203
1204
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1205
1206
1207
1208
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1209
                            delta=delta_message,
1210
                            logprobs=logprobs,
1211
                            finish_reason=None,
1212
1213
1214
1215
1216
1217
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1218
1219

                    # if the model is finished generating
1220
                    else:
1221
1222
1223
1224
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1225
1226
1227
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1228
                        # only happens if we are NOT using structured outputs
1229
                        auto_tools_called = False
1230
                        if tool_parser:
1231
1232
1233
1234
1235
1236
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1237
1238
1239
                        else:
                            index = 0

1240
1241
1242
1243
1244
1245
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1246
                            latest_delta_len = 0
1247
1248
                            if (
                                isinstance(
1249
                                    delta_message.tool_calls[0].function,
1250
1251
1252
1253
1254
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1255
                                latest_delta_len = len(
1256
1257
                                    delta_message.tool_calls[0].function.arguments
                                )
1258

1259
1260
1261
1262
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1263
1264
1265
1266
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1267

1268
                            # get what we've streamed so far for arguments
1269
                            # for the current tool
1270
1271
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1272
                                actual_call = actual_call[:-latest_delta_len]
1273
1274

                            # check to see if there's anything left to stream
1275
                            remaining_call = expected_call.replace(actual_call, "", 1)
1276
                            # set that as a delta message
1277
1278
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1279
                            )
1280

1281
                        # Send the finish response for each request.n only once
1282
1283
1284
1285
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1286
1287
                        if (
                            auto_tools_called
1288
                            or (tools_streamed[i] and not tool_choice_function_name)
1289
1290
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1291
1292
                            finish_reason_ = "tool_calls"
                        else:
1293
1294
1295
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1296
1297
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1298
                            delta=delta_message,
1299
                            logprobs=logprobs,
1300
                            finish_reason=finish_reason_,
1301
                            stop_reason=output.stop_reason,
1302
1303
1304
1305
1306
1307
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1308

1309
                        finish_reason_sent[i] = True
1310

1311
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1312
1313
1314
1315
1316
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1317
1318
                        model=model_name,
                    )
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1329
                    data = chunk.model_dump_json(exclude_unset=True)
1330
1331
                    yield f"data: {data}\n\n"

1332
1333
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1334
1335
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1336
1337
1338
1339
1340
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1341
1342
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1343
1344
                        cached_tokens=num_cached_tokens
                    )
1345
1346
1347
1348
1349
1350
1351

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1352
1353
1354
1355
1356
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1357
                yield f"data: {final_usage_data}\n\n"
1358

1359
1360
1361
1362
1363
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1364
1365
1366
1367
1368
1369
1370
1371
1372
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1373
1374
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1375
1376
1377
1378
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1379
                        output_token_ids=None,  # Consider also logging all token IDs
1380
1381
1382
1383
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1384

1385
1386
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1387
        except Exception as e:
1388
            logger.exception("Error in chat completion stream generator.")
1389
            data = self.create_streaming_error_response(e)
1390
            yield f"data: {data}\n\n"
1391
1392
1393
1394
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1395
1396
1397
1398
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1399
        model_name: str,
1400
        conversation: list[ConversationMessage],
1401
        tokenizer: TokenizerLike,
1402
        request_metadata: RequestResponseMetadata,
1403
    ) -> ErrorResponse | ChatCompletionResponse:
1404
1405
        from vllm.tokenizers.mistral import MistralTokenizer

1406
        created_time = int(time.time())
1407
        final_res: RequestOutput | None = None
1408

1409
1410
1411
1412
1413
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1414
        except ValueError as e:
1415
            return self.create_error_response(e)
1416

1417
1418
        assert final_res is not None

1419
        choices: list[ChatCompletionResponseChoice] = []
1420
        if self.tool_call_id_type == "kimi_k2":
1421
1422
1423
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1424

1425
1426
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1427
1428
1429
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1430
            token_ids = output.token_ids
1431
            out_logprobs = output.logprobs
1432
            tool_call_info = None
1433

1434
1435
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1436
                logprobs = self._create_chat_logprobs(
1437
                    token_ids=token_ids,
1438
                    top_logprobs=out_logprobs,
1439
                    num_output_top_logprobs=request.top_logprobs,
1440
                    tokenizer=tokenizer,
1441
                    return_as_token_id=request.return_tokens_as_token_ids,
1442
1443
1444
                )
            else:
                logprobs = None
1445
1446

            if self.use_harmony:
1447
                reasoning, content, _ = parse_chat_output(token_ids)
1448
                if not request.include_reasoning:
1449
                    reasoning = None
1450

1451
                if self.tool_parser is not None:
1452
1453
1454
1455
1456
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1457
1458
1459
1460
1461
1462
1463
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1464
                    content = tool_call_info.content
1465
1466
                    message = ChatMessage(
                        role=role,
1467
                        reasoning=reasoning,
1468
1469
1470
1471
1472
1473
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1474
                        reasoning=reasoning,
1475
1476
                        content=content,
                    )
1477
1478
1479
1480
1481

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1482
1483
1484
1485
1486
1487
1488
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1489
                    stop_reason=output.stop_reason,
1490
1491
1492
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1493
1494
1495
                )
                choices.append(choice_data)
                continue
1496

1497
            if self.reasoning_parser:
1498
                try:
1499
1500
1501
1502
1503
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1504
1505
1506
1507
1508
                    # Pass the same chat template kwargs as used in tokenization
                    chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                        request.chat_template_kwargs,
                        self.default_chat_template_kwargs,
                    )
1509
1510
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
1511
                        chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
1512
                    )
1513
1514
1515
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1516
1517
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1518
                reasoning, content = reasoning_parser.extract_reasoning(
1519
1520
                    output.text, request=request
                )
1521
                if not request.include_reasoning:
1522
                    reasoning = None
1523
            else:
1524
                reasoning = None
1525
                content = output.text
1526

1527
            auto_tools_called = False
1528
1529
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
            if self.use_harmony:
                # Harmony models already have parsed content and tool_calls
                # through parse_chat_output. Respect its output directly.
                message = ChatMessage(
                    role=role,
                    reasoning=reasoning,
                    content=content,
                    tool_calls=tool_calls if tool_calls else [],
                )

            elif (not self.enable_auto_tools or not self.tool_parser) and (
1551
1552
1553
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1554
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1555

1556
1557
1558
1559
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1560
                assert tool_calls is not None and len(tool_calls) > 0
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
                tool_call_class_items = []
                for idx, tc in enumerate(tool_calls):
                    # Use native ID if available (e.g., Kimi K2),
                    # otherwise generate ID with correct id_type
                    if tc.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tc.id, function=tc)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(tool_call_class(function=tc))
                        else:
                            generated_id = make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tc.name,
1579
                                idx=history_tool_call_cnt,
1580
1581
1582
1583
1584
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tc)
                            )
                    history_tool_call_cnt += 1
1585
1586
                message = ChatMessage(
                    role=role,
1587
                    reasoning=reasoning,
1588
                    content="",
1589
                    tool_calls=tool_call_class_items,
1590
                )
1591

1592
            elif request.tool_choice and request.tool_choice == "required":
1593
1594
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
                for idx, tool_call in enumerate(tool_calls):
                    # Use native ID if available,
                    # otherwise generate ID with correct id_type
                    if tool_call.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tool_call.id, function=tool_call)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(
                                tool_call_class(function=tool_call)
                            )
                        else:
                            generated_id = make_tool_call_id(
1612
1613
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
1614
                                idx=history_tool_call_cnt,
1615
1616
1617
1618
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tool_call)
                            )
1619
                    history_tool_call_cnt += 1
1620
1621
1622
                message = ChatMessage(
                    role=role,
                    content="",
1623
                    tool_calls=tool_call_class_items,
1624
                    reasoning=reasoning,
1625
                )
1626

1627
1628
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1629
            elif not request.tool_choice or request.tool_choice == "none":
1630
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1631
1632

            # handle when there are tools and tool choice is auto
1633
1634
1635
1636
1637
1638
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1639
1640
1641
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1642
1643
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
                    tool_call_items = []
                    for idx, tc in enumerate(tool_calls):
                        # Use native ID if available (e.g., Kimi K2),
                        # otherwise generate ID with correct id_type
                        if tc.id:
                            tool_call_items.append(
                                tool_call_class(id=tc.id, function=tc)
                            )
                        else:
                            # Generate ID using the correct format (kimi_k2 or random),
                            # but leave it to the class if it's Mistral to preserve
                            # 9-char IDs
                            if isinstance(tokenizer, MistralTokenizer):
                                tool_call_items.append(tool_call_class(function=tc))
                            else:
                                generated_id = make_tool_call_id(
                                    id_type=self.tool_call_id_type,
                                    func_name=tc.name,
1662
                                    idx=history_tool_call_cnt,
1663
1664
1665
1666
1667
                                )
                                tool_call_items.append(
                                    tool_call_class(id=generated_id, function=tc)
                                )
                        history_tool_call_cnt += 1
1668
1669
                    message = ChatMessage(
                        role=role,
1670
                        reasoning=reasoning,
1671
                        content=content,
1672
                        tool_calls=tool_call_items,
1673
                    )
1674
1675
1676
1677

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1678
1679
1680
1681
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1682
1683
                    if content and len(content) > 0:
                        ret_content = content
1684
1685
                    message = ChatMessage(
                        role=role,
1686
                        reasoning=reasoning,
1687
1688
                        content=ret_content,
                    )
1689
1690
1691
1692
1693
1694

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1695
1696
                    "completion."
                )
1697
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1698
1699
1700
1701
1702
1703
1704
1705
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1706

1707
1708
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1709
                message=message,
1710
                logprobs=logprobs,
1711
1712
1713
1714
1715
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1716
                stop_reason=output.stop_reason,
1717
1718
1719
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1720
            )
1721
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1722

1723
1724
            choices.append(choice_data)

1725
        if request.echo:
1726
            last_msg_content: str | list[dict[str, str]] = ""
1727
1728
1729
1730
1731
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1732
                last_msg_content = conversation[-1]["content"] or ""
1733
            if isinstance(last_msg_content, list):
1734
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1735
1736

            for choice in choices:
1737
                full_message = last_msg_content + (choice.message.content or "")
1738
1739
                choice.message.content = full_message

1740
        assert final_res.prompt_token_ids is not None
1741
        num_prompt_tokens = len(final_res.prompt_token_ids)
1742
1743
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1744
        num_generated_tokens = sum(
1745
1746
1747
1748
1749
1750
1751
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1752
1753
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1754
1755
                cached_tokens=final_res.num_cached_tokens
            )
1756
1757
1758

        request_metadata.final_usage_info = usage

1759
1760
1761
1762
1763
1764
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1765
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1766
1767
1768
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1769
            kv_transfer_params=final_res.kv_transfer_params,
1770
1771
        )

1772
1773
1774
1775
1776
1777
1778
1779
1780
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1781
1782
1783
1784
1785
                    for tc in choice.message.tool_calls:  # type: ignore
                        function_call: FunctionCall = tc.function  # type: ignore
                        tool_call_descriptions.append(
                            f"{function_call.name}({function_call.arguments})"
                        )
1786
1787
1788
1789
1790
1791
1792
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1793
                        output_token_ids = final_res.outputs[choice.index].token_ids
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1804
        return response
1805
1806

    def _get_top_logprobs(
1807
1808
        self,
        logprobs: dict[int, Logprob],
1809
        top_logprobs: int | None,
1810
        tokenizer: TokenizerLike | None,
1811
1812
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1813
        return [
1814
            ChatCompletionLogProb(
1815
1816
1817
1818
1819
1820
1821
1822
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1823
1824
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1825
1826
            )
            for i, p in enumerate(logprobs.items())
1827
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1828
1829
1830
1831
1832
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1833
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1834
        tokenizer: TokenizerLike | None,
1835
1836
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1837
1838
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1839
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1840

1841
1842
1843
1844
1845
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1846
1847
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1848
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1849
                if should_return_as_token_id:
1850
                    token = f"token_id:{token_id}"
1851
                else:
1852
1853
                    if tokenizer is None:
                        raise ValueError(
1854
                            "Unable to get tokenizer because `skip_tokenizer_init=True`"
1855
1856
                        )

1857
                    token = tokenizer.decode(token_id)
1858

1859
1860
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1861
                        token=token,
1862
                        bytes=list(token.encode("utf-8", errors="replace")),
1863
1864
                    )
                )
1865
            else:
1866
1867
1868
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1869
1870
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1871
                        token=self._get_decoded_token(
1872
1873
1874
                            step_token,
                            token_id,
                            tokenizer,
1875
                            should_return_as_token_id,
1876
1877
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1878
1879
1880
1881
1882
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1883
                        top_logprobs=self._get_top_logprobs(
1884
1885
1886
1887
1888
1889
1890
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1891
1892

        return ChatCompletionLogProbs(content=logprobs_content)
1893

1894
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1895
1896
1897
1898
1899
1900
1901
1902
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1903
1904
1905
1906
1907
1908
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1909
1910
1911

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1912
        delta_message: DeltaMessage | None,
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1924
            output.finish_reason is not None
1925
1926
1927
1928
1929
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1930
1931
1932
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1933

1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )

1963
1964
1965
    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1966
        should_include_tools: bool = True,
1967
1968
1969
    ):
        messages: list[OpenAIMessage] = []

1970
1971
1972
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
1973
        maybe_serialize_tool_calls(request)  # type: ignore[arg-type]
1974

1975
1976
1977
1978
1979
1980
1981
1982
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1983
            python_description=None,
1984
            with_custom_tools=should_include_tools,
1985
        )
1986
1987
1988
        messages.append(sys_msg)

        # Add developer message.
1989
1990
        if request.tools:
            dev_msg = get_developer_message(
1991
                tools=request.tools if should_include_tools else None  # type: ignore[arg-type]
1992
1993
            )
            messages.append(dev_msg)
1994
1995

        # Add user message.
1996
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1997
1998
1999

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
2000
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
2001
2002
2003
2004
2005

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

2006
        return messages, [engine_prompt]