serving_chat.py 80.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Any, Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
25
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.parser.harmony_utils import (
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
30
    parse_chat_inputs_to_harmony_messages,
31
32
33
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
54
55
56
from vllm.entrypoints.openai.serving_chat_stream_harmony import (
    extract_harmony_streaming_delta,
)
57
58
59
60
61
from vllm.entrypoints.openai.serving_engine import (
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
62
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
63
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
64
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
65
from vllm.inputs.data import TokensPrompt
66
from vllm.logger import init_logger
67
from vllm.logprobs import Logprob
68
from vllm.outputs import CompletionOutput, RequestOutput
69
from vllm.sampling_params import BeamSearchParams, SamplingParams
70
71
72
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
73
74
75
76
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
77
78
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
79
from vllm.utils.collection_utils import as_list
80
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
81
82
83
84
85

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
86
87
88
    def __init__(
        self,
        engine_client: EngineClient,
89
        models: OpenAIServingModels,
90
91
        response_role: str,
        *,
92
93
        request_logger: RequestLogger | None,
        chat_template: str | None,
94
        chat_template_content_format: ChatTemplateContentFormatOption,
95
        trust_request_chat_template: bool = False,
96
        return_tokens_as_token_ids: bool = False,
97
        reasoning_parser: str = "",
98
        enable_auto_tools: bool = False,
99
        exclude_tools_when_tool_choice_none: bool = False,
100
        tool_parser: str | None = None,
101
        enable_prompt_tokens_details: bool = False,
102
        enable_force_include_usage: bool = False,
103
        enable_log_outputs: bool = False,
104
        log_error_stack: bool = False,
105
        default_chat_template_kwargs: dict[str, Any] | None = None,
106
    ) -> None:
107
108
109
110
111
112
113
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
114

115
        self.response_role = response_role
116
117
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
118
        self.trust_request_chat_template = trust_request_chat_template
119
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
120
        self.enable_log_outputs = enable_log_outputs
121

122
123
124
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

125
126
127
128
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
129
130
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
131
132
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
133
134
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
135

136
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
137
        self.enable_force_include_usage = enable_force_include_usage
138
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
139
        if self.default_sampling_params:
140
141
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
142
143
144
145
146
147
148
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
149
        else:
150
            self.tool_call_id_type = "random"
151

152
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
153
154
155
156
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
157
158
                get_stop_tokens_for_assistant_actions()
            )
159
160
161
162
163
164
165
166
167
168
169

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Get the tokenizer from the engine
            tokenizer = await self.engine_client.get_tokenizer()

            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                tokenizer,
                dummy_request.messages,
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                add_generation_prompt=True,
                continue_final_message=False,
                tool_dicts=None,
                documents=None,
                chat_template_kwargs=None,
208
                default_chat_template_kwargs=self.default_chat_template_kwargs,
209
210
211
212
213
214
215
216
217
218
219
                tool_parser=None,
                add_special_tokens=False,
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

220
    async def create_chat_completion(
221
222
        self,
        request: ChatCompletionRequest,
223
224
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
225
226
        """
        Chat Completion API similar to OpenAI's API.
227

228
229
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
230
        Chat Completion API.
231
232
233
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
234
            logger.error("Error with model %s", error_check_ret)
235
236
            return error_check_ret

237
238
239
240
241
242
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

243
        try:
244
            lora_request = self._maybe_get_adapters(
245
246
                request, supports_default_mm_loras=True
            )
247

248
            model_name = self.models.model_name(lora_request)
249

250
            tokenizer = await self.engine_client.get_tokenizer()
251

252
253
            tool_parser = self.tool_parser

254
            if isinstance(tokenizer, MistralTokenizer):
255
256
257
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
258
                maybe_serialize_tool_calls(request)
259
                truncate_tool_call_ids(request)
260
                validate_request_params(request)
261

262
263
264
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
265
266
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
267
268
269
270
271
272
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
273
            ):
274
275
276
277
278
279
280
281
282
283
284
285
286
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
287

288
289
290
291
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
292
293
294
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
295

296
297
            if not self.use_harmony:
                # Common case.
298
299
300
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
301
                    trust_request_chat_template=self.trust_request_chat_template,
302
303
304
                )
                if error_check_ret is not None:
                    return error_check_ret
305
                conversation, engine_prompts = await self._preprocess_chat(
306
307
308
                    request,
                    tokenizer,
                    request.messages,
309
                    chat_template=request.chat_template or self.chat_template,
310
                    chat_template_content_format=self.chat_template_content_format,
311
312
313
314
315
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
316
                    default_chat_template_kwargs=self.default_chat_template_kwargs,
317
318
319
320
321
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
322
323
324
325
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
326
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
327
            logger.exception("Error in preprocessing prompt inputs")
328
            return self.create_error_response(f"{e} {e.__cause__}")
329

330
331
332
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
333
334
335
336
337

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

338
339
340
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

341
        # Schedule the request and get the result generator.
342
        generators: list[AsyncGenerator[RequestOutput, None]] = []
343
        try:
344
            for i, engine_prompt in enumerate(engine_prompts):
345
                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
346
347
348
349
350
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
351
352
353
354
355
356
357
358

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
359
360
                    default_sampling_params=self.default_sampling_params,
                )
361

362
                sampling_params: SamplingParams | BeamSearchParams
363
364
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
365
366
                        max_tokens, self.default_sampling_params
                    )
367
368
                else:
                    sampling_params = request.to_sampling_params(
369
370
371
372
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
373
374
375
376
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
377

378
                self._log_inputs(
379
                    sub_request_id,
380
                    engine_prompt,
381
382
383
                    params=sampling_params,
                    lora_request=lora_request,
                )
384

385
386
387
388
389
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
390
391

                if isinstance(sampling_params, BeamSearchParams):
392
                    generator = self.beam_search(
393
                        prompt=engine_prompt,
394
                        request_id=sub_request_id,
395
                        params=sampling_params,
396
                        lora_request=lora_request,
397
                        trace_headers=trace_headers,
398
399
                    )
                else:
400
                    engine_request, tokenization_kwargs = await self._process_inputs(
401
                        sub_request_id,
402
403
404
405
406
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
407
                        data_parallel_rank=data_parallel_rank,
408
                    )
409

410
                    generator = self.engine_client.generate(
411
                        engine_request,
412
                        sampling_params,
413
                        sub_request_id,
414
415
416
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
417
418
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
419
                        data_parallel_rank=data_parallel_rank,
420
421
422
                    )

                generators.append(generator)
423
        except ValueError as e:
424
            return self.create_error_response(e)
425

426
        assert len(generators) == 1
427
        (result_generator,) = generators
428

429
430
431
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
432
433
434
435
436
437
438
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
439
            )
440

441
442
        try:
            return await self.chat_completion_full_generator(
443
444
445
446
447
448
449
450
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
451
452
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
453
        except ValueError as e:
454
            return self.create_error_response(e)
455
456
457
458

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
459
        return request.messages[-1]["role"]
460

461
    @staticmethod
462
    def _bracket_level(s: str, opening="{", closing="}") -> int:
463
464
465
466
467
468
469
470
471
472
473
474
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
475
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
476
477
478
479
480
481
482
483
484
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
485
            if c == "{":
486
487
                bracket_level += 1
                passed_zero = bracket_level == 0
488
            elif c == "}":
489
490
491
492
493
494
495
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
496
                if c == ",":
497
498
499
500
501
502
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
503
        current_text: str | None,
504
505
        delta_text: str,
        function_name_returned: bool,
506
507
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
508
509
510
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
511
512
513
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
514
            logger.debug("not enough tokens to parse into JSON yet")
515
516
517
518
519
520
521
522
523
524
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
525
526
                delta_text, previous_text
            )
527
528
529
530
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
531
532
533
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
534
535
536
537
538
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
539
540
541
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
542
543
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
544
545
                        arguments, previous_text
                    )
546
547
548
549

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
550
                    if finishes_previous_tool and "parameters" not in current_tool_call:
551
552
553
                        current_tool_call = obj[-2]

                    function_name_returned = True
554
555
556
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
557
558
559
560
561
562
563
564
565
566
567
568
569
570
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
571
572
573

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
574
575
                        delta_text, previous_text
                    )
576
577

                    if delta_text != "":
578
579
580
581
582
583
584
585
586
587
588
589
590
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
591
592
593
594
595
                    else:
                        delta_message = None

        return delta_message, function_name_returned

596
    async def chat_completion_stream_generator(
597
598
599
600
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
601
        model_name: str,
602
        conversation: list[ConversationMessage],
603
        tokenizer: TokenizerLike | None,
604
        request_metadata: RequestResponseMetadata,
605
    ) -> AsyncGenerator[str, None]:
606
        created_time = int(time.time())
607
        chunk_object_type: Final = "chat.completion.chunk"
608
        first_iteration = True
609
610

        # Send response for each token for each request.n (index)
611
612
613
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
614
        num_prompt_tokens = 0
615
        num_cached_tokens = None
616
617
        if self.use_harmony:
            harmony_parsers = [
618
                get_streamable_parser_for_assistant() for _ in range(num_choices)
619
            ]
620
621
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
622
623
624
625
626
627
628
629
630

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
631
632
            and self._should_stream_with_auto_tool_parsing(request)
        )
633

634
        all_previous_token_ids: list[list[int]] | None
635
        function_name_returned = [False] * num_choices
636
        if self.tool_call_id_type == "kimi_k2":
637
638
639
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
640

641
642
643
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

644
645
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
646
        if tool_choice_auto or self.reasoning_parser:
647
648
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
649
650
651
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
652
        else:
653
            all_previous_token_ids = None
654

655
        try:
656
            if self.reasoning_parser:
657
658
659
660
661
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

662
663
664
665
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
666
667
668
669
670
671
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
672
673
674
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
675
676
677
678
679
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

680
                tool_parsers: list[ToolParser | None] = [
681
682
683
684
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
685
        except Exception as e:
686
            logger.exception("Error in tool parser creation.")
687
            data = self.create_streaming_error_response(e)
688
689
690
691
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

692
        stream_options = request.stream_options
693
694
695
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
696

697
698
        try:
            async for res in result_generator:
699
700
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
701
702
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
703

704
705
706
707
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
708
                    num_cached_tokens = res.num_cached_tokens
709
710
                    # Send first response for each request.n (index) with
                    # the role
711
                    role = self.get_chat_request_role(request)
712
713
714

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
715
                    for i in range(num_choices):
716
717
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
718
719
720
721
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
722
                            logprobs=None,
723
724
                            finish_reason=None,
                        )
725
726

                        # return prompt_token_ids at the first chunk ever
727
728
729
730
731
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
732
                            model=model_name,
733
734
735
736
737
738
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
739

740
741
742
743
744
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
745
746
                                total_tokens=num_prompt_tokens,
                            )
747

748
749
750
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

751
752
                    # Send response to echo the input portion of the
                    # last message
753
                    if request.echo:
754
                        last_msg_content: str | list[dict[str, str]] = ""
755
756
757
758
759
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
760
                            last_msg_content = conversation[-1]["content"] or ""
761
762

                        if last_msg_content:
763
                            for i in range(num_choices):
764
765
766
767
768
769
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
770
771
772
773
774
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
775
776
                                    model=model_name,
                                )
777
778
779
780
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
781
782
                                        total_tokens=num_prompt_tokens,
                                    )
783

784
                                data = chunk.model_dump_json(exclude_unset=True)
785
786
787
788
789
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
790
                    tool_parser = tool_parsers[i]
791
792
793
794

                    if finish_reason_sent[i]:
                        continue

795
                    if request.logprobs and request.top_logprobs is not None:
796
                        assert output.logprobs is not None, "Did not output logprobs"
797
                        logprobs = self._create_chat_logprobs(
798
799
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
800
                            tokenizer=tokenizer,
801
                            num_output_top_logprobs=request.top_logprobs,
802
                            return_as_token_id=request.return_tokens_as_token_ids,
803
804
805
806
                        )
                    else:
                        logprobs = None

807
808
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
809
                        prev_recipient = harmony_parser.current_recipient
810
                        delta_text = ""
811
812
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
813
                            delta_text += harmony_parser.last_content_delta or ""
814
815
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
816
817
818
819
820
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
821
822
                    else:
                        delta_text = output.text
823

824
825
826
827
828
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
829
830
831
                        # Chunked prefill case, don't return empty chunks
                        continue

832
                    delta_message: DeltaMessage | None
833

834
                    # just update previous_texts and previous_token_ids
835
                    if tool_choice_auto or self.reasoning_parser:
836
837
838
839
840
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
841
842
                        # avoid the None + list error.
                        if previous_token_ids:
843
                            current_token_ids = previous_token_ids + as_list(
844
845
                                output.token_ids
                            )
846
                        else:
847
                            current_token_ids = as_list(output.token_ids)
848

849
                    if self.use_harmony:
850
851
852
853
854
855
856
857
858
859
860
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
                                cur_channel=cur_channel,
                                cur_recipient=cur_recipient,
                                prev_recipient=prev_recipient,
                                delta_text=delta_text,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
861
                    # handle streaming deltas for tools with named tool_choice
862
                    elif tool_choice_function_name:
863
864
865
866
867
868
869
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
870
871
                            assert reasoning_parser is not None
                            delta_message = (
872
                                reasoning_parser.extract_reasoning_streaming(
873
874
875
876
877
878
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
879
880
                                )
                            )
881
882
883
884
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
885
                            # Only keep 'content', remove 'reasoning'.
886
                            if reasoning_parser.is_reasoning_end(
887
888
889
890
891
892
893
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
894
                                reasoning_end_arr[i] = True
895
896
897
898
899
900
901
902
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
903
                            if self.reasoning_parser:
904
905
906
                                delta_text = previous_text + delta_text
                                current_text = ""

907
908
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
909
910
911
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
912
913
                            else:
                                delta_tool_call = DeltaToolCall(
914
                                    id=make_tool_call_id(),
915
916
917
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
918
919
920
921
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
922
923
                                function_name_returned[i] = True

924
925
926
927
928
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
929
                            tools_streamed[i] = True
930

931
932
933
934
935
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
936
937
938
939
940
941
942
943
944
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
945

946
947
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
948
                                reasoning_parser.extract_reasoning_streaming(
949
950
951
952
953
954
955
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
956
                            )
957
958
959
960
961
962
963
964
965
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

966
                        else:
967
                            # either finished reasoning or no reasoning at all
968
                            content = current_text
969
970
971
972
973
974
975
976
977

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
978
                            )
979
980
981
982
983
984
985
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
986

987
988
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
989
                    elif tool_choice_auto and self.reasoning_parser:
990
991
992
993
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
994
                        output_token_ids = as_list(output.token_ids)
995
                        if not reasoning_end_arr[i]:
996
997
998
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
999
1000
1001
1002
1003
1004
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
1005
                                reasoning_end_arr[i] = True
1006
                                current_token_ids = output_token_ids
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1017
1018
                                    )
                                )
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1036
1037

                        # handle tool calls only after reasoning is done,
1038
                        if reasoning_end_arr[i]:
1039
                            delta_token_ids = output_token_ids
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1050
                            delta_message = tool_parser.extract_tool_calls_streaming(
1051
1052
                                previous_text=previous_text,
                                current_text=current_text,
1053
                                delta_text=delta_text,
1054
1055
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1073
1074
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1075

1076
                    # when only reasoning
1077
                    elif self.reasoning_parser:
1078
1079
1080
1081
1082
1083
1084
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1085
                        )
1086
                    # handle streaming just a content delta
1087
1088
1089
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1090
                    # update the previous values for the next iteration
1091
1092
1093
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1094
1095
1096
1097
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1098
1099
1100
1101
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1102

1103
                    # set the previous values for the next iteration
1104
                    previous_num_tokens[i] += len(output.token_ids)
1105
1106
1107
1108
1109
1110

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1111
1112
1113
1114
1115
1116
1117
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1118
                            continue
1119
                        delta_message = DeltaMessage()
1120

1121
1122
1123
1124
1125
1126
1127
1128
1129
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1130
1131
                                if tc.function and tc.function.arguments
                            )
1132
1133
1134
1135
1136

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1137
                                output_token_ids=as_list(output.token_ids),
1138
1139
1140
1141
1142
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1143
1144
1145
1146
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1147
                            delta=delta_message,
1148
                            logprobs=logprobs,
1149
                            finish_reason=None,
1150
1151
1152
1153
1154
1155
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1156
1157

                    # if the model is finished generating
1158
                    else:
1159
1160
1161
1162
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1163
1164
1165
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1166
                        # only happens if we are NOT using structured outputs
1167
                        auto_tools_called = False
1168
                        if tool_parser:
1169
1170
1171
1172
1173
1174
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1175
1176
1177
                        else:
                            index = 0

1178
1179
1180
1181
1182
1183
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1184
                            latest_delta_len = 0
1185
1186
                            if (
                                isinstance(
1187
                                    delta_message.tool_calls[0].function,
1188
1189
1190
1191
1192
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1193
                                latest_delta_len = len(
1194
1195
                                    delta_message.tool_calls[0].function.arguments
                                )
1196

1197
1198
1199
1200
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1201
1202
1203
1204
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1205

1206
                            # get what we've streamed so far for arguments
1207
                            # for the current tool
1208
1209
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1210
                                actual_call = actual_call[:-latest_delta_len]
1211
1212

                            # check to see if there's anything left to stream
1213
                            remaining_call = expected_call.replace(actual_call, "", 1)
1214
                            # set that as a delta message
1215
1216
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1217
                            )
1218

1219
                        # Send the finish response for each request.n only once
1220
1221
1222
1223
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1224
1225
                        if (
                            auto_tools_called
1226
                            or (tools_streamed[i] and not tool_choice_function_name)
1227
1228
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1229
1230
                            finish_reason_ = "tool_calls"
                        else:
1231
1232
1233
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1234
1235
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1236
                            delta=delta_message,
1237
                            logprobs=logprobs,
1238
                            finish_reason=finish_reason_,
1239
                            stop_reason=output.stop_reason,
1240
1241
1242
1243
1244
1245
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1246

1247
                        finish_reason_sent[i] = True
1248

1249
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1250
1251
1252
1253
1254
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1255
1256
                        model=model_name,
                    )
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1267
                    data = chunk.model_dump_json(exclude_unset=True)
1268
1269
                    yield f"data: {data}\n\n"

1270
1271
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1272
1273
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1274
1275
1276
1277
1278
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1279
1280
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1281
1282
                        cached_tokens=num_cached_tokens
                    )
1283
1284
1285
1286
1287
1288
1289

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1290
1291
1292
1293
1294
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1295
                yield f"data: {final_usage_data}\n\n"
1296

1297
1298
1299
1300
1301
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1302
1303
1304
1305
1306
1307
1308
1309
1310
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1311
1312
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1313
1314
1315
1316
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1317
                        output_token_ids=None,  # Consider also logging all token IDs
1318
1319
1320
1321
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1322

1323
1324
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1325
        except Exception as e:
1326
            logger.exception("Error in chat completion stream generator.")
1327
            data = self.create_streaming_error_response(e)
1328
            yield f"data: {data}\n\n"
1329
1330
1331
1332
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1333
1334
1335
1336
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1337
        model_name: str,
1338
        conversation: list[ConversationMessage],
1339
        tokenizer: TokenizerLike | None,
1340
        request_metadata: RequestResponseMetadata,
1341
    ) -> ErrorResponse | ChatCompletionResponse:
1342
        created_time = int(time.time())
1343
        final_res: RequestOutput | None = None
1344

1345
1346
1347
1348
1349
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1350
        except ValueError as e:
1351
            return self.create_error_response(e)
1352

1353
1354
        assert final_res is not None

1355
        choices: list[ChatCompletionResponseChoice] = []
1356
        if self.tool_call_id_type == "kimi_k2":
1357
1358
1359
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1360

1361
1362
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1363
1364
1365
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1366
            token_ids = output.token_ids
1367
            out_logprobs = output.logprobs
1368
            tool_call_info = None
1369

1370
1371
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1372
                logprobs = self._create_chat_logprobs(
1373
                    token_ids=token_ids,
1374
                    top_logprobs=out_logprobs,
1375
                    num_output_top_logprobs=request.top_logprobs,
1376
                    tokenizer=tokenizer,
1377
                    return_as_token_id=request.return_tokens_as_token_ids,
1378
1379
1380
                )
            else:
                logprobs = None
1381
1382

            if self.use_harmony:
1383
                reasoning, content, _ = parse_chat_output(token_ids)
1384
                if not request.include_reasoning:
1385
                    reasoning = None
1386

1387
                if self.tool_parser is not None:
1388
1389
1390
1391
1392
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1393
1394
1395
1396
1397
1398
1399
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1400
                    content = tool_call_info.content
1401
1402
                    message = ChatMessage(
                        role=role,
1403
                        reasoning=reasoning,
1404
1405
1406
1407
1408
1409
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1410
                        reasoning=reasoning,
1411
1412
                        content=content,
                    )
1413
1414
1415
1416
1417

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1418
1419
1420
1421
1422
1423
1424
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1425
                    stop_reason=output.stop_reason,
1426
1427
1428
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1429
1430
1431
                )
                choices.append(choice_data)
                continue
1432

1433
            if self.reasoning_parser:
1434
                try:
1435
1436
1437
1438
1439
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1440
1441
1442
1443
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1444
1445
1446
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1447
1448
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1449
                reasoning, content = reasoning_parser.extract_reasoning(
1450
1451
                    output.text, request=request
                )
1452
                if not request.include_reasoning:
1453
                    reasoning = None
1454
            else:
1455
                reasoning = None
1456
                content = output.text
1457

1458
            auto_tools_called = False
1459
1460
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1471
1472
1473
1474
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1475
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1476
1477

            # if the request uses tools and specified a tool choice
1478
1479
1480
1481
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1482
                assert tool_calls is not None and len(tool_calls) > 0
1483
1484
                message = ChatMessage(
                    role=role,
1485
                    reasoning=reasoning,
1486
                    content="",
1487
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1488
                )
1489

1490
            elif request.tool_choice and request.tool_choice == "required":
1491
1492
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1493
                for tool_call in tool_calls:
1494
1495
1496
1497
1498
1499
1500
1501
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1502
1503
                        )
                    )
1504
                    history_tool_call_cnt += 1
1505
1506
1507
                message = ChatMessage(
                    role=role,
                    content="",
1508
                    tool_calls=tool_call_class_items,
1509
                    reasoning=reasoning,
1510
                )
1511

1512
1513
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1514
            elif not request.tool_choice or request.tool_choice == "none":
1515
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1516
1517

            # handle when there are tools and tool choice is auto
1518
1519
1520
1521
1522
1523
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1524
1525
1526
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1527
1528
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1529
1530
                    message = ChatMessage(
                        role=role,
1531
                        reasoning=reasoning,
1532
1533
1534
1535
1536
1537
1538
1539
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1540
                    )
1541
1542
1543
1544

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1545
1546
1547
1548
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1549
1550
                    if content and len(content) > 0:
                        ret_content = content
1551
1552
                    message = ChatMessage(
                        role=role,
1553
                        reasoning=reasoning,
1554
1555
                        content=ret_content,
                    )
1556
1557
1558
1559
1560
1561

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1562
1563
                    "completion."
                )
1564
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1565
1566
1567
1568
1569
1570
1571
1572
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1573

1574
1575
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1576
                message=message,
1577
                logprobs=logprobs,
1578
1579
1580
1581
1582
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1583
                stop_reason=output.stop_reason,
1584
1585
1586
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1587
            )
1588
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1589

1590
1591
            choices.append(choice_data)

1592
        if request.echo:
1593
            last_msg_content: str | list[dict[str, str]] = ""
1594
1595
1596
1597
1598
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1599
                last_msg_content = conversation[-1]["content"] or ""
1600
            if isinstance(last_msg_content, list):
1601
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1602
1603

            for choice in choices:
1604
                full_message = last_msg_content + (choice.message.content or "")
1605
1606
                choice.message.content = full_message

1607
        assert final_res.prompt_token_ids is not None
1608
        num_prompt_tokens = len(final_res.prompt_token_ids)
1609
1610
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1611
        num_generated_tokens = sum(
1612
1613
1614
1615
1616
1617
1618
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1619
1620
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1621
1622
                cached_tokens=final_res.num_cached_tokens
            )
1623
1624
1625

        request_metadata.final_usage_info = usage

1626
1627
1628
1629
1630
1631
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1632
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1633
1634
1635
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1636
            kv_transfer_params=final_res.kv_transfer_params,
1637
1638
        )

1639
1640
1641
1642
1643
1644
1645
1646
1647
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1648
1649
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1650
1651
                            tc.function, "arguments"
                        ):
1652
                            tool_call_descriptions.append(
1653
1654
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1655
1656
1657
1658
1659
1660
1661
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1662
                        output_token_ids = final_res.outputs[choice.index].token_ids
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1673
        return response
1674
1675

    def _get_top_logprobs(
1676
1677
        self,
        logprobs: dict[int, Logprob],
1678
        top_logprobs: int | None,
1679
        tokenizer: TokenizerLike | None,
1680
1681
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1682
        return [
1683
            ChatCompletionLogProb(
1684
1685
1686
1687
1688
1689
1690
1691
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1692
1693
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1694
1695
            )
            for i, p in enumerate(logprobs.items())
1696
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1697
1698
1699
1700
1701
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1702
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1703
        tokenizer: TokenizerLike | None,
1704
1705
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1706
1707
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1708
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1709

1710
1711
1712
1713
1714
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1715
1716
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1717
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1718
                if should_return_as_token_id:
1719
                    token = f"token_id:{token_id}"
1720
                else:
1721
1722
1723
1724
1725
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1726
                    token = tokenizer.decode(token_id)
1727

1728
1729
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1730
                        token=token,
1731
                        bytes=list(token.encode("utf-8", errors="replace")),
1732
1733
                    )
                )
1734
            else:
1735
1736
1737
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1738
1739
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1740
                        token=self._get_decoded_token(
1741
1742
1743
                            step_token,
                            token_id,
                            tokenizer,
1744
                            should_return_as_token_id,
1745
1746
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1747
1748
1749
1750
1751
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1752
                        top_logprobs=self._get_top_logprobs(
1753
1754
1755
1756
1757
1758
1759
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1760
1761

        return ChatCompletionLogProbs(content=logprobs_content)
1762

1763
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1764
1765
1766
1767
1768
1769
1770
1771
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1772
1773
1774
1775
1776
1777
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1778
1779
1780

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1781
        delta_message: DeltaMessage | None,
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1793
            output.finish_reason is not None
1794
1795
1796
1797
1798
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1799
1800
1801
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1802

1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )

1832
1833
1834
    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1835
        should_include_tools: bool = True,
1836
1837
1838
    ):
        messages: list[OpenAIMessage] = []

1839
1840
1841
1842
1843
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1844
1845
1846
1847
1848
1849
1850
1851
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1852
            python_description=None,
1853
            with_custom_tools=should_include_tools,
1854
        )
1855
1856
1857
        messages.append(sys_msg)

        # Add developer message.
1858
1859
1860
1861
1862
        if request.tools:
            dev_msg = get_developer_message(
                tools=request.tools if should_include_tools else None
            )
            messages.append(dev_msg)
1863
1864

        # Add user message.
1865
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1866
1867
1868

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1869
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1870
1871
1872
1873
1874

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1875
        return messages, [engine_prompt]