serving.py 79.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from http import HTTPStatus
10
from typing import TYPE_CHECKING, Any, Final
11

12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from partial_json_parser.core.options import Allow
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
22
    get_tool_call_id_type,
23
24
    make_tool_call_id,
)
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.chat_completion.protocol import (
27
28
29
30
31
32
33
34
35
36
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
37
38
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
39
    TokenState,
40
41
42
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
43
44
45
46
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
47
    FunctionCall,
48
49
50
51
52
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
53
from vllm.entrypoints.openai.engine.serving import (
54
55
56
57
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
58
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
59
60
61
62
63
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    parse_chat_output,
)
64
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
65
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
66
from vllm.inputs.data import ProcessorInputs
67
from vllm.logger import init_logger
68
from vllm.logprobs import Logprob
69
from vllm.outputs import CompletionOutput, RequestOutput
70
from vllm.parser import ParserManager
71
from vllm.reasoning import ReasoningParser
72
from vllm.renderers import ChatParams
73
from vllm.sampling_params import BeamSearchParams, SamplingParams
74
from vllm.tokenizers import TokenizerLike
75
76
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
77
from vllm.tool_parsers.utils import partial_json_loads
78
from vllm.utils.collection_utils import as_list
79
from vllm.utils.mistral import is_mistral_tokenizer
80
81
82

if TYPE_CHECKING:
    from vllm.entrypoints.serve.render.serving import OpenAIServingRender
83
84
85
86
87

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
88
89
90
    def __init__(
        self,
        engine_client: EngineClient,
91
        models: OpenAIServingModels,
92
93
        response_role: str,
        *,
94
        openai_serving_render: "OpenAIServingRender",
95
96
        request_logger: RequestLogger | None,
        chat_template: str | None,
97
        chat_template_content_format: ChatTemplateContentFormatOption,
98
        trust_request_chat_template: bool = False,
99
        return_tokens_as_token_ids: bool = False,
100
        reasoning_parser: str = "",
101
        enable_auto_tools: bool = False,
102
        exclude_tools_when_tool_choice_none: bool = False,
103
        tool_parser: str | None = None,
104
        enable_prompt_tokens_details: bool = False,
105
        enable_force_include_usage: bool = False,
106
        enable_log_outputs: bool = False,
107
        enable_log_deltas: bool = True,
108
        default_chat_template_kwargs: dict[str, Any] | None = None,
109
    ) -> None:
110
111
112
113
114
115
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
        )
116

117
        self.openai_serving_render = openai_serving_render
118
        self.response_role = response_role
119
120
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
121
        self.trust_request_chat_template = trust_request_chat_template
122
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
123
        self.enable_log_outputs = enable_log_outputs
124
        self.enable_log_deltas = enable_log_deltas
125

126
        # set up reasoning parser
127
        self.reasoning_parser_cls = ParserManager.get_reasoning_parser(
128
129
            reasoning_parser_name=reasoning_parser
        )
130
131
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
132
133
134
135
        self.tool_parser = ParserManager.get_tool_parser(
            tool_parser_name=tool_parser,
            enable_auto_tools=enable_auto_tools,
            model_name=self.model_config.model,
136
137
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
138

139
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
140
        self.enable_force_include_usage = enable_force_include_usage
141
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
142
143
144
145
146
147
        mc = self.model_config
        self.override_max_tokens = (
            self.default_sampling_params.get("max_tokens")
            if mc.generation_config not in ("auto", "vllm")
            else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
        )
148
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
149
150
151
152
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
153
154
                get_stop_tokens_for_assistant_actions()
            )
155

156
        self.tool_call_id_type = get_tool_call_id_type(self.model_config)
157

158
159
160
161
162
163
164
165
166
167
        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

168
169
170
171
172
173
    def warmup(self) -> None:
        self.renderer.warmup(
            ChatParams(
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                chat_template_kwargs=self.default_chat_template_kwargs,
174
            )
175
        )
176

177
    async def render_chat_request(
178
179
        self,
        request: ChatCompletionRequest,
180
    ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
181
        """
182
183
184
185
        Validate the model and preprocess a chat completion request.

        Delegates preprocessing logic to OpenAIServingRender, adding the
        engine-aware checks (LoRA model validation, engine health).
186

187
188
189
        Returns:
            A tuple of (conversation, engine_prompts) on success,
            or an ErrorResponse on failure.
190
191
192
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
193
            logger.error("Error with model %s", error_check_ret)
194
195
            return error_check_ret

196
197
198
199
200
201
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

202
        return await self.openai_serving_render.render_chat(request)
203
204
205
206
207
208
209
210
211
212
213
214
215

    async def create_chat_completion(
        self,
        request: ChatCompletionRequest,
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
        """
        Chat Completion API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
        Chat Completion API.
        """
216
217
218
219
        # Streaming response
        tokenizer = self.renderer.tokenizer
        assert tokenizer is not None
        reasoning_parser: ReasoningParser | None = None
220
221
222
223
224
225
226
227
228
229
        if self.reasoning_parser_cls:
            # Pass the same chat template kwargs as used in tokenization
            chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                request.chat_template_kwargs,
                self.default_chat_template_kwargs,
            )
            reasoning_parser = self.reasoning_parser_cls(
                tokenizer,
                chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
            )
230
231
232
233
234
        result = await self.render_chat_request(request)
        if isinstance(result, ErrorResponse):
            return result

        conversation, engine_prompts = result
235

236
237
238
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
239
240
241
242
243

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

244
        lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
245

246
        model_name = self.models.model_name(lora_request)
247

248
249
250
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

251
        # Schedule the request and get the result generator.
252
        max_model_len = self.model_config.max_model_len
253
        generators: list[AsyncGenerator[RequestOutput, None]] = []
254
255
256
257
258
259
260
261
        for i, engine_prompt in enumerate(engine_prompts):
            prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids

            # If we are creating sub requests for multiple prompts, ensure that they
            # have unique request ids.
            sub_request_id = (
                request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
            )
262

263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
            max_tokens = get_max_tokens(
                max_model_len,
                request.max_completion_tokens
                if request.max_completion_tokens is not None
                else request.max_tokens,
                self._extract_prompt_len(engine_prompt),
                self.default_sampling_params,
                self.override_max_tokens,
            )

            sampling_params: SamplingParams | BeamSearchParams
            if request.use_beam_search:
                sampling_params = request.to_beam_search_params(
                    max_tokens, self.default_sampling_params
                )
            else:
                sampling_params = request.to_sampling_params(
                    max_tokens,
281
                    self.default_sampling_params,
282
                )
283

284
285
286
287
288
289
            self._log_inputs(
                sub_request_id,
                engine_prompt,
                params=sampling_params,
                lora_request=lora_request,
            )
290

291
292
293
294
295
296
297
298
299
300
            trace_headers = (
                None
                if raw_request is None
                else await self._get_trace_headers(raw_request.headers)
            )

            if isinstance(sampling_params, BeamSearchParams):
                generator = self.beam_search(
                    prompt=engine_prompt,
                    request_id=sub_request_id,
301
302
                    params=sampling_params,
                    lora_request=lora_request,
303
                    trace_headers=trace_headers,
304
                )
305
            else:
306
307
308
309
310
311
312
313
                if not request.include_reasoning:
                    reasoning_ended = True
                elif reasoning_parser:
                    reasoning_ended = reasoning_parser.is_reasoning_end(
                        prompt_token_ids or []
                    )
                else:
                    reasoning_ended = None
314

315
316
317
318
319
320
321
322
323
324
                generator = self.engine_client.generate(
                    engine_prompt,
                    sampling_params,
                    sub_request_id,
                    lora_request=lora_request,
                    trace_headers=trace_headers,
                    priority=request.priority,
                    data_parallel_rank=data_parallel_rank,
                    reasoning_ended=reasoning_ended,
                )
325

326
            generators.append(generator)
327

328
        assert len(generators) == 1
329
        (result_generator,) = generators
330

331
332
        if request.stream:
            return self.chat_completion_stream_generator(
333
334
335
336
337
338
339
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
340
                reasoning_parser,
341
            )
342

343
344
345
346
347
348
349
350
351
352
        return await self.chat_completion_full_generator(
            request,
            result_generator,
            request_id,
            model_name,
            conversation,
            tokenizer,
            request_metadata,
            reasoning_parser,
        )
353
354
355
356

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
357
        return request.messages[-1]["role"]
358

359
    @staticmethod
360
    def _bracket_level(s: str, opening="{", closing="}") -> int:
361
362
363
364
365
366
367
368
369
370
371
372
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
373
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
374
375
376
377
378
379
380
381
382
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
383
            if c == "{":
384
385
                bracket_level += 1
                passed_zero = bracket_level == 0
386
            elif c == "}":
387
388
389
390
391
392
393
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
394
                if c == ",":
395
396
397
398
399
400
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
401
        current_text: str | None,
402
403
        delta_text: str,
        function_name_returned: bool,
404
405
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
406
407
408
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
409
        try:
410
411
412
413
414
415
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
416
            logger.debug("not enough tokens to parse into JSON yet")
417
418
419
420
421
422
423
424
425
426
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
427
428
                delta_text, previous_text
            )
429
430
431
432
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
433
434
435
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
436
437
438
439
440
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
441
442
443
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
444
445
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
446
447
                        arguments, previous_text
                    )
448
449
450
451

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
452
                    if finishes_previous_tool and "parameters" not in current_tool_call:
453
454
455
                        current_tool_call = obj[-2]

                    function_name_returned = True
456
457
458
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
459
460
461
462
463
464
465
466
467
468
469
470
471
472
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
473
474
475

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
476
477
                        delta_text, previous_text
                    )
478
479

                    if delta_text != "":
480
481
482
483
484
485
486
487
488
489
490
491
492
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
493
494
495
496
497
                    else:
                        delta_message = None

        return delta_message, function_name_returned

498
    async def chat_completion_stream_generator(
499
500
501
502
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
503
        model_name: str,
504
        conversation: list[ConversationMessage],
505
        tokenizer: TokenizerLike,
506
        request_metadata: RequestResponseMetadata,
507
        reasoning_parser: ReasoningParser | None = None,
508
    ) -> AsyncGenerator[str, None]:
509
        created_time = int(time.time())
510
        chunk_object_type: Final = "chat.completion.chunk"
511
        first_iteration = True
512
513

        # Send response for each token for each request.n (index)
514
515
516
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
517
        num_prompt_tokens = 0
518
        num_cached_tokens = None
519
520
        if self.use_harmony:
            harmony_parsers = [
521
                get_streamable_parser_for_assistant() for _ in range(num_choices)
522
            ]
523
524
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
525
526
527
528
529
530
531
532
533

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
534
535
            and self._should_stream_with_auto_tool_parsing(request)
        )
536

537
        all_previous_token_ids: list[list[int]] | None
538
        function_name_returned = [False] * num_choices
539
        if self.tool_call_id_type == "kimi_k2":
540
541
542
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
543

544
545
546
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

547
548
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
549
        if tool_choice_auto or reasoning_parser:
550
551
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
552
553
554
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
555
            prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
556
        else:
557
            all_previous_token_ids = None
558

559
560
561
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
562
563
564
565
566
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

567
                tool_parsers: list[ToolParser | None] = [
568
569
570
571
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
572
        except Exception as e:
573
            logger.exception("Error in tool parser creation.")
574
            data = self.create_streaming_error_response(e)
575
576
577
578
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

579
        stream_options = request.stream_options
580
581
582
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
583

584
585
        try:
            async for res in result_generator:
586
587
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
588
589
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
590

591
592
593
594
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
595
                    num_cached_tokens = res.num_cached_tokens
596
597
                    # Send first response for each request.n (index) with
                    # the role
598
                    role = self.get_chat_request_role(request)
599
600
601

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
602
                    for i in range(num_choices):
603
604
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
605
606
607
608
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
609
                            logprobs=None,
610
611
                            finish_reason=None,
                        )
612
613

                        # return prompt_token_ids at the first chunk ever
614
615
616
617
618
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
619
                            model=model_name,
620
621
622
623
624
625
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
626

627
628
629
630
631
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
632
633
                                total_tokens=num_prompt_tokens,
                            )
634

635
636
637
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

638
639
                    # Send response to echo the input portion of the
                    # last message
640
                    if request.echo:
641
                        last_msg_content: str | list[dict[str, str]] = ""
642
643
644
645
646
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
647
                            last_msg_content = conversation[-1]["content"] or ""
648
649

                        if last_msg_content:
650
                            for i in range(num_choices):
651
652
653
654
655
656
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
657
658
659
660
661
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
662
663
                                    model=model_name,
                                )
664
665
666
667
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
668
669
                                        total_tokens=num_prompt_tokens,
                                    )
670

671
                                data = chunk.model_dump_json(exclude_unset=True)
672
673
674
675
676
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
677
                    tool_parser = tool_parsers[i]
678

679
                    if (
680
                        reasoning_parser
681
682
683
684
685
686
687
688
                        and res.prompt_token_ids
                        and prompt_is_reasoning_end_arr[i] is None
                    ):
                        # only check once per choice, because prompt_token_ids
                        # are the same for all deltas in that choice
                        prompt_is_reasoning_end_arr[i] = (
                            reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        )
689
690
691
                    if finish_reason_sent[i]:
                        continue

692
                    if request.logprobs and request.top_logprobs is not None:
693
                        assert output.logprobs is not None, "Did not output logprobs"
694
                        logprobs = self._create_chat_logprobs(
695
696
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
697
                            tokenizer=tokenizer,
698
                            num_output_top_logprobs=request.top_logprobs,
699
                            return_as_token_id=request.return_tokens_as_token_ids,
700
701
702
703
                        )
                    else:
                        logprobs = None

704
705
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
706
                        prev_recipient = harmony_parser.current_recipient
707
708
709

                        # Track accumulated content per token with their state
                        token_states: list[TokenState] = []
710
711
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
712
713
714
715
716
717
718
719
720
                            token_delta = harmony_parser.last_content_delta or ""
                            token_states.append(
                                TokenState(
                                    harmony_parser.current_channel,
                                    harmony_parser.current_recipient,
                                    token_delta,
                                )
                            )
                        delta_text = "".join(delta for _, _, delta in token_states)
721
                        cur_channel = harmony_parser.current_channel
722

723
724
725
726
727
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
728
729
                    else:
                        delta_text = output.text
730

731
732
733
734
735
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
736
737
738
                        # Chunked prefill case, don't return empty chunks
                        continue

739
                    delta_message: DeltaMessage | None
740

741
                    # just update previous_texts and previous_token_ids
742
                    if tool_choice_auto or reasoning_parser:
743
744
745
746
747
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
748
749
                        # avoid the None + list error.
                        if previous_token_ids:
750
                            current_token_ids = previous_token_ids + as_list(
751
752
                                output.token_ids
                            )
753
                        else:
754
                            current_token_ids = as_list(output.token_ids)
755

756
                    if self.use_harmony:
757
758
759
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
760
                                token_states=token_states,
761
762
763
764
765
                                prev_recipient=prev_recipient,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
766
                    # handle streaming deltas for tools with named tool_choice
767
                    elif tool_choice_function_name:
768
769
770
771
772
773
774
775
776
777
778
                        # When encountering think end id in prompt_token_ids
                        # i.e {"enable_thinking": False},
                        # check BEFORE calling the parser to avoid a spurious
                        # reasoning delta on the first chunk.
                        if (
                            reasoning_parser
                            and not reasoning_end_arr[i]
                            and prompt_is_reasoning_end_arr[i]
                        ):
                            reasoning_end_arr[i] = True

779
                        if (
780
                            reasoning_parser
781
782
783
784
785
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
786
787
                            assert reasoning_parser is not None
                            delta_message = (
788
                                reasoning_parser.extract_reasoning_streaming(
789
790
791
792
793
794
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
795
796
                                )
                            )
797
                            # When encountering think end id in delta_token_ids,
798
                            # set reasoning status to end.
799
                            # Only keep 'content', remove 'reasoning'.
800
801
                            if reasoning_parser.is_reasoning_end(
                                as_list(output.token_ids)
802
                            ):
803
                                reasoning_end_arr[i] = True
804
805
806
807
808
809
810
811
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
812
                            if reasoning_parser:
813
814
815
                                delta_text = previous_text + delta_text
                                current_text = ""

816
817
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
818
819
820
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
821
                            else:
822
                                # Generate ID based on tokenizer type
823
                                if is_mistral_tokenizer(tokenizer):
824
825
826
827
828
829
830
                                    tool_call_id = MistralToolCall.generate_random_id()
                                else:
                                    tool_call_id = make_tool_call_id(
                                        id_type=self.tool_call_id_type,
                                        func_name=tool_choice_function_name,
                                        idx=history_tool_call_cnt,
                                    )
831
                                delta_tool_call = DeltaToolCall(
832
                                    id=tool_call_id,
833
834
835
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
836
837
838
839
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
840
                                function_name_returned[i] = True
841
                                history_tool_call_cnt += 1
842

843
844
845
846
847
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
848
                            tools_streamed[i] = True
849

850
851
852
853
854
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
855
856
857
                        output_token_ids = as_list(output.token_ids)

                        if (
858
                            reasoning_parser is not None
859
                            and not reasoning_end_arr[i]
860
                            and prompt_is_reasoning_end_arr[i]
861
862
                        ):
                            reasoning_end_arr[i] = True
863

864
                        if reasoning_parser and not reasoning_end_arr[i]:
865
                            delta_message = (
866
                                reasoning_parser.extract_reasoning_streaming(
867
868
869
870
871
872
873
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
874
                            )
875
876
877
878
879
880
881
882
883
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

884
                        else:
885
                            # either finished reasoning or no reasoning at all
886
                            content = current_text
887
888
889
890
891
892
893
894
895

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
896
                            )
897
898
899
900
901
902
903
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
904

905
906
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
907
                    elif tool_choice_auto and reasoning_parser:
908
909
910
                        assert tool_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
911
                        output_token_ids = as_list(output.token_ids)
912
                        if not reasoning_end_arr[i]:
913
914
915
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
916
                            if prompt_is_reasoning_end_arr[i]:
917
                                reasoning_end_arr[i] = True
918
                                current_token_ids = output_token_ids
919
920
921
922
923
924
925
926
927
928
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
929
930
                                    )
                                )
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
948
949

                        # handle tool calls only after reasoning is done,
950
                        if reasoning_end_arr[i]:
951
                            delta_token_ids = output_token_ids
952
953
954
955
956
957
958
959
960
961
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

962
                            delta_message = tool_parser.extract_tool_calls_streaming(
963
964
                                previous_text=previous_text,
                                current_text=current_text,
965
                                delta_text=delta_text,
966
967
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
985
986
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
987

988
                    # when only reasoning
989
                    elif reasoning_parser:
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
                        # When encountering think end id in prompt_token_ids
                        # i.e {"enable_thinking": False},
                        # set reasoning status to end.
                        # Route all generated tokens as content directly.
                        if prompt_is_reasoning_end_arr[i]:
                            delta_message = DeltaMessage(content=delta_text)
                        else:
                            delta_message = (
                                reasoning_parser.extract_reasoning_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                )
                            )
1007
                    # handle streaming just a content delta
1008
1009
1010
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1011
                    # update the previous values for the next iteration
1012
                    if (tool_choice_auto or reasoning_parser) and not self.use_harmony:
1013
1014
1015
1016
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1017
1018
1019
1020
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1021

1022
                    # set the previous values for the next iteration
1023
                    previous_num_tokens[i] += len(output.token_ids)
1024
1025
1026
1027
1028
1029

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1030
1031
1032
1033
1034
1035
1036
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1037
                            continue
1038
                        delta_message = DeltaMessage()
1039

1040
1041
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
1042
                        delta_content_parts = []
1043
                        if delta_message.content:
1044
                            delta_content_parts.append(delta_message.content)
1045
1046
                        if delta_message.reasoning:
                            reasoning = delta_message.reasoning
1047
1048
1049
                            delta_content_parts.append(f"[reasoning: {reasoning}]")
                        if delta_message.tool_calls:
                            tool_args = "".join(
1050
1051
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1052
1053
                                if tc.function and tc.function.arguments
                            )
1054
1055
                            if tool_args:
                                delta_content_parts.append(f"[tool_calls: {tool_args}]")
1056

1057
1058
                        if delta_content_parts and self.enable_log_deltas:
                            delta_content = " ".join(delta_content_parts)
1059
1060
1061
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1062
                                output_token_ids=as_list(output.token_ids),
1063
1064
1065
1066
1067
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1068
1069
1070
1071
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1072
                            delta=delta_message,
1073
                            logprobs=logprobs,
1074
                            finish_reason=None,
1075
1076
1077
1078
1079
1080
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1081
1082

                    # if the model is finished generating
1083
                    else:
1084
1085
1086
1087
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1088
1089
1090
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1091
                        # only happens if we are NOT using structured outputs
1092
                        auto_tools_called = False
1093
                        if tool_parser:
1094
1095
1096
1097
1098
1099
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1100
1101
1102
                        else:
                            index = 0

1103
1104
1105
1106
1107
1108
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1109
                            latest_delta_len = 0
1110
1111
                            if (
                                isinstance(
1112
                                    delta_message.tool_calls[0].function,
1113
1114
1115
1116
1117
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1118
                                latest_delta_len = len(
1119
1120
                                    delta_message.tool_calls[0].function.arguments
                                )
1121

1122
                            # get the expected call based on partial JSON
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
                            # parsing which "autocompletes" the JSON.
                            # Tool parsers (e.g. Qwen3Coder) store
                            # arguments as a JSON string in
                            # prev_tool_call_arr. Calling json.dumps()
                            # on an already-serialized string would
                            # double-serialize it (e.g. '{"k":1}' becomes
                            # '"{\\"k\\":1}"'), which then causes the
                            # replace() below to fail and append the
                            # entire double-serialized string as a
                            # spurious final delta.
                            args = tool_parser.prev_tool_call_arr[index].get(
                                "arguments", {}
1135
                            )
1136
1137
1138
1139
                            if isinstance(args, str):
                                expected_call = args
                            else:
                                expected_call = json.dumps(args, ensure_ascii=False)
1140

1141
                            # get what we've streamed so far for arguments
1142
                            # for the current tool
1143
1144
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1145
                                actual_call = actual_call[:-latest_delta_len]
1146
1147

                            # check to see if there's anything left to stream
1148
                            remaining_call = expected_call.replace(actual_call, "", 1)
1149
                            # set that as a delta message
1150
1151
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1152
                            )
1153

1154
                        # Send the finish response for each request.n only once
1155
1156
1157
1158
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1159
1160
                        if (
                            auto_tools_called
1161
                            or (tools_streamed[i] and not tool_choice_function_name)
1162
1163
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1164
1165
                            finish_reason_ = "tool_calls"
                        else:
1166
1167
1168
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1169
1170
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1171
                            delta=delta_message,
1172
                            logprobs=logprobs,
1173
                            finish_reason=finish_reason_,
1174
                            stop_reason=output.stop_reason,
1175
1176
1177
1178
1179
1180
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1181

1182
                        finish_reason_sent[i] = True
1183

1184
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1185
1186
1187
1188
1189
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1190
1191
                        model=model_name,
                    )
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1202
                    data = chunk.model_dump_json(exclude_unset=True)
1203
1204
                    yield f"data: {data}\n\n"

1205
1206
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1207
1208
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1209
1210
1211
1212
1213
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1214
1215
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1216
1217
                        cached_tokens=num_cached_tokens
                    )
1218
1219
1220
1221
1222
1223
1224

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1225
1226
1227
1228
1229
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1230
                yield f"data: {final_usage_data}\n\n"
1231

1232
1233
1234
1235
1236
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1237
1238
1239
1240
1241
1242
1243
1244
1245
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1246
1247
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1248
1249
1250
1251
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1252
                        output_token_ids=None,  # Consider also logging all token IDs
1253
1254
1255
1256
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1257

1258
1259
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1260
        except Exception as e:
1261
            logger.exception("Error in chat completion stream generator.")
1262
            data = self.create_streaming_error_response(e)
1263
            yield f"data: {data}\n\n"
1264
1265
1266
1267
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1268
1269
1270
1271
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1272
        model_name: str,
1273
        conversation: list[ConversationMessage],
1274
        tokenizer: TokenizerLike,
1275
        request_metadata: RequestResponseMetadata,
1276
        reasoning_parser: ReasoningParser | None = None,
1277
    ) -> ErrorResponse | ChatCompletionResponse:
1278
1279
        from vllm.tokenizers.mistral import MistralTokenizer

1280
        created_time = int(time.time())
1281
        final_res: RequestOutput | None = None
1282

1283
1284
1285
1286
1287
1288
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")

1289
1290
1291
1292
1293
1294
        if final_res is None:
            return self.create_error_response(
                "No output received from the engine.",
                err_type="InternalServerError",
                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
            )
1295

1296
        choices: list[ChatCompletionResponseChoice] = []
1297
        if self.tool_call_id_type == "kimi_k2":
1298
1299
1300
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1301

1302
1303
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1304
1305
1306
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1307
            token_ids = output.token_ids
1308
            out_logprobs = output.logprobs
1309
            tool_call_info = None
1310

1311
1312
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1313
                logprobs = self._create_chat_logprobs(
1314
                    token_ids=token_ids,
1315
                    top_logprobs=out_logprobs,
1316
                    num_output_top_logprobs=request.top_logprobs,
1317
                    tokenizer=tokenizer,
1318
                    return_as_token_id=request.return_tokens_as_token_ids,
1319
1320
1321
                )
            else:
                logprobs = None
1322
1323

            if self.use_harmony:
1324
                reasoning, content, _ = parse_chat_output(token_ids)
1325
                if not request.include_reasoning:
1326
                    reasoning = None
1327

1328
                if self.tool_parser is not None:
1329
1330
1331
1332
1333
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1334
1335
1336
1337
1338
1339
1340
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1341
                    content = tool_call_info.content
1342
1343
                    message = ChatMessage(
                        role=role,
1344
                        reasoning=reasoning,
1345
1346
1347
1348
1349
1350
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1351
                        reasoning=reasoning,
1352
1353
                        content=content,
                    )
1354
1355
1356
1357
1358

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1359
1360
1361
1362
1363
1364
1365
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1366
                    stop_reason=output.stop_reason,
1367
1368
1369
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1370
1371
1372
                )
                choices.append(choice_data)
                continue
1373

1374
            if reasoning_parser:
1375
1376
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1377
                reasoning, content = reasoning_parser.extract_reasoning(
1378
1379
                    output.text, request=request
                )
1380
                if not request.include_reasoning:
1381
                    reasoning = None
1382
            else:
1383
                reasoning = None
1384
                content = output.text
1385

1386
            auto_tools_called = False
1387
1388
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1389
1390
1391
1392
1393
1394
1395
1396
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
1397
                MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall
1398
            )
1399
            if (not self.enable_auto_tools or not self.tool_parser) and (
1400
1401
1402
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1403
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1404

1405
1406
1407
1408
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1409
                assert tool_calls is not None and len(tool_calls) > 0
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
                tool_call_class_items = []
                for idx, tc in enumerate(tool_calls):
                    # Use native ID if available (e.g., Kimi K2),
                    # otherwise generate ID with correct id_type
                    if tc.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tc.id, function=tc)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(tool_call_class(function=tc))
                        else:
                            generated_id = make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tc.name,
1428
                                idx=history_tool_call_cnt,
1429
1430
1431
1432
1433
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tc)
                            )
                    history_tool_call_cnt += 1
1434
1435
                message = ChatMessage(
                    role=role,
1436
                    reasoning=reasoning,
1437
                    content="",
1438
                    tool_calls=tool_call_class_items,
1439
                )
1440

1441
            elif request.tool_choice and request.tool_choice == "required":
1442
                tool_call_class_items = []
1443
                tool_calls = tool_calls or []
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
                for idx, tool_call in enumerate(tool_calls):
                    # Use native ID if available,
                    # otherwise generate ID with correct id_type
                    if tool_call.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tool_call.id, function=tool_call)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(
                                tool_call_class(function=tool_call)
                            )
                        else:
                            generated_id = make_tool_call_id(
1461
1462
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
1463
                                idx=history_tool_call_cnt,
1464
1465
1466
1467
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tool_call)
                            )
1468
                    history_tool_call_cnt += 1
1469
1470
1471
                message = ChatMessage(
                    role=role,
                    content="",
1472
                    tool_calls=tool_call_class_items,
1473
                    reasoning=reasoning,
1474
                )
1475

1476
1477
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1478
            elif not request.tool_choice or request.tool_choice == "none":
1479
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1480
1481

            # handle when there are tools and tool choice is auto
1482
1483
1484
1485
1486
1487
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1488
1489
1490
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1491
1492
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
                    tool_call_items = []
                    for idx, tc in enumerate(tool_calls):
                        # Use native ID if available (e.g., Kimi K2),
                        # otherwise generate ID with correct id_type
                        if tc.id:
                            tool_call_items.append(
                                tool_call_class(id=tc.id, function=tc)
                            )
                        else:
                            # Generate ID using the correct format (kimi_k2 or random),
                            # but leave it to the class if it's Mistral to preserve
                            # 9-char IDs
                            if isinstance(tokenizer, MistralTokenizer):
                                tool_call_items.append(tool_call_class(function=tc))
                            else:
                                generated_id = make_tool_call_id(
                                    id_type=self.tool_call_id_type,
                                    func_name=tc.name,
1511
                                    idx=history_tool_call_cnt,
1512
1513
1514
1515
1516
                                )
                                tool_call_items.append(
                                    tool_call_class(id=generated_id, function=tc)
                                )
                        history_tool_call_cnt += 1
1517
1518
                    message = ChatMessage(
                        role=role,
1519
                        reasoning=reasoning,
1520
                        content=content,
1521
                        tool_calls=tool_call_items,
1522
                    )
1523
1524
1525
1526

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1527
1528
1529
1530
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1531
1532
                    if content and len(content) > 0:
                        ret_content = content
1533
1534
                    message = ChatMessage(
                        role=role,
1535
                        reasoning=reasoning,
1536
1537
                        content=ret_content,
                    )
1538
1539
1540
1541
1542
1543

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1544
1545
                    "completion."
                )
1546
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1547
1548
1549
1550
1551
1552
1553
1554
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1555

1556
1557
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1558
                message=message,
1559
                logprobs=logprobs,
1560
1561
1562
1563
1564
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1565
                stop_reason=output.stop_reason,
1566
1567
1568
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1569
            )
1570
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1571

1572
1573
            choices.append(choice_data)

1574
        if request.echo:
1575
            last_msg_content: str | list[dict[str, str]] = ""
1576
1577
1578
1579
1580
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1581
                last_msg_content = conversation[-1]["content"] or ""
1582
            if isinstance(last_msg_content, list):
1583
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1584
1585

            for choice in choices:
1586
                full_message = last_msg_content + (choice.message.content or "")
1587
1588
                choice.message.content = full_message

1589
        assert final_res.prompt_token_ids is not None
1590
        num_prompt_tokens = len(final_res.prompt_token_ids)
1591
1592
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1593
        num_generated_tokens = sum(
1594
1595
1596
1597
1598
1599
1600
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1601
1602
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1603
1604
                cached_tokens=final_res.num_cached_tokens
            )
1605
1606
1607

        request_metadata.final_usage_info = usage

1608
1609
1610
1611
1612
1613
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1614
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1615
1616
1617
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1618
            kv_transfer_params=final_res.kv_transfer_params,
1619
1620
        )

1621
1622
1623
1624
1625
1626
1627
1628
1629
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1630
1631
1632
1633
1634
                    for tc in choice.message.tool_calls:  # type: ignore
                        function_call: FunctionCall = tc.function  # type: ignore
                        tool_call_descriptions.append(
                            f"{function_call.name}({function_call.arguments})"
                        )
1635
1636
1637
1638
1639
1640
1641
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1642
                        output_token_ids = final_res.outputs[choice.index].token_ids
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1653
        return response
1654
1655

    def _get_top_logprobs(
1656
1657
        self,
        logprobs: dict[int, Logprob],
1658
        top_logprobs: int | None,
1659
        tokenizer: TokenizerLike | None,
1660
1661
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1662
        return [
1663
            ChatCompletionLogProb(
1664
1665
1666
1667
1668
1669
1670
1671
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1672
1673
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1674
1675
            )
            for i, p in enumerate(logprobs.items())
1676
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1677
1678
1679
1680
1681
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1682
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1683
        tokenizer: TokenizerLike | None,
1684
1685
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1686
1687
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1688
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1689

1690
1691
1692
1693
1694
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1695
1696
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1697
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1698
                if should_return_as_token_id:
1699
                    token = f"token_id:{token_id}"
1700
                else:
1701
1702
                    if tokenizer is None:
                        raise ValueError(
1703
                            "Unable to get tokenizer because `skip_tokenizer_init=True`"
1704
1705
                        )

1706
                    token = tokenizer.decode(token_id)
1707

1708
1709
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1710
                        token=token,
1711
                        bytes=list(token.encode("utf-8", errors="replace")),
1712
1713
                    )
                )
1714
            else:
1715
1716
1717
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1718
1719
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1720
                        token=self._get_decoded_token(
1721
1722
1723
                            step_token,
                            token_id,
                            tokenizer,
1724
                            should_return_as_token_id,
1725
1726
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1727
1728
1729
1730
1731
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1732
                        top_logprobs=self._get_top_logprobs(
1733
1734
1735
1736
1737
1738
1739
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1740
1741

        return ChatCompletionLogProbs(content=logprobs_content)
1742

1743
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1744
1745
1746
1747
1748
1749
1750
1751
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1752
1753
1754
1755
1756
1757
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1758
1759
1760

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1761
        delta_message: DeltaMessage | None,
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1773
            output.finish_reason is not None
1774
1775
1776
1777
1778
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1779
1780
1781
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1782

1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )