serving.py 79.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from http import HTTPStatus
10
from typing import TYPE_CHECKING, Any, Final
11

12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from partial_json_parser.core.options import Allow
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
from vllm.entrypoints.logger import RequestLogger
25
from vllm.entrypoints.openai.chat_completion.protocol import (
26
27
28
29
30
31
32
33
34
35
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
36
37
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
38
    TokenState,
39
40
41
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
42
43
44
45
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
46
    FunctionCall,
47
48
49
50
51
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
52
from vllm.entrypoints.openai.engine.serving import (
53
54
55
56
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
57
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
58
59
60
61
62
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    parse_chat_output,
)
63
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
64
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
65
from vllm.inputs.data import ProcessorInputs
66
from vllm.logger import init_logger
67
from vllm.logprobs import Logprob
68
from vllm.outputs import CompletionOutput, RequestOutput
69
from vllm.parser import ParserManager
70
from vllm.reasoning import ReasoningParser
71
from vllm.renderers import ChatParams
72
from vllm.sampling_params import BeamSearchParams, SamplingParams
73
from vllm.tokenizers import TokenizerLike
74
75
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
76
from vllm.tool_parsers.utils import partial_json_loads
77
from vllm.utils.collection_utils import as_list
78
from vllm.utils.mistral import is_mistral_tokenizer
79
80
81

if TYPE_CHECKING:
    from vllm.entrypoints.serve.render.serving import OpenAIServingRender
82
83
84
85
86

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
87
88
89
    def __init__(
        self,
        engine_client: EngineClient,
90
        models: OpenAIServingModels,
91
92
        response_role: str,
        *,
93
        openai_serving_render: "OpenAIServingRender",
94
95
        request_logger: RequestLogger | None,
        chat_template: str | None,
96
        chat_template_content_format: ChatTemplateContentFormatOption,
97
        trust_request_chat_template: bool = False,
98
        return_tokens_as_token_ids: bool = False,
99
        reasoning_parser: str = "",
100
        enable_auto_tools: bool = False,
101
        exclude_tools_when_tool_choice_none: bool = False,
102
        tool_parser: str | None = None,
103
        enable_prompt_tokens_details: bool = False,
104
        enable_force_include_usage: bool = False,
105
        enable_log_outputs: bool = False,
106
        enable_log_deltas: bool = True,
107
        default_chat_template_kwargs: dict[str, Any] | None = None,
108
    ) -> None:
109
110
111
112
113
114
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
        )
115

116
        self.openai_serving_render = openai_serving_render
117
        self.response_role = response_role
118
119
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
120
        self.trust_request_chat_template = trust_request_chat_template
121
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
122
        self.enable_log_outputs = enable_log_outputs
123
        self.enable_log_deltas = enable_log_deltas
124

125
        # set up reasoning parser
126
        self.reasoning_parser_cls = ParserManager.get_reasoning_parser(
127
128
            reasoning_parser_name=reasoning_parser
        )
129
130
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
131
132
133
134
        self.tool_parser = ParserManager.get_tool_parser(
            tool_parser_name=tool_parser,
            enable_auto_tools=enable_auto_tools,
            model_name=self.model_config.model,
135
136
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
137

138
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
139
        self.enable_force_include_usage = enable_force_include_usage
140
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
141
142
143
144
145
146
        mc = self.model_config
        self.override_max_tokens = (
            self.default_sampling_params.get("max_tokens")
            if mc.generation_config not in ("auto", "vllm")
            else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
        )
147
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
148
149
150
151
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
152
153
                get_stop_tokens_for_assistant_actions()
            )
154

155
156
157
158
159
160
161
162
163
164
        # Handle tool call ID type for Kimi K2 (supporting test mocking via overrides)
        hf_overrides = getattr(self.model_config, "hf_overrides", None)
        if self.model_config.hf_text_config.model_type == "kimi_k2" or (
            isinstance(hf_overrides, dict)
            and hf_overrides.get("model_type") == "kimi_k2"
        ):
            self.tool_call_id_type = "kimi_k2"
        else:
            self.tool_call_id_type = "random"

165
166
167
168
169
170
171
172
173
174
        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

175
176
177
178
179
180
    def warmup(self) -> None:
        self.renderer.warmup(
            ChatParams(
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                chat_template_kwargs=self.default_chat_template_kwargs,
181
            )
182
        )
183

184
    async def render_chat_request(
185
186
        self,
        request: ChatCompletionRequest,
187
    ) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
188
        """
189
190
191
192
        Validate the model and preprocess a chat completion request.

        Delegates preprocessing logic to OpenAIServingRender, adding the
        engine-aware checks (LoRA model validation, engine health).
193

194
195
196
        Returns:
            A tuple of (conversation, engine_prompts) on success,
            or an ErrorResponse on failure.
197
198
199
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
200
            logger.error("Error with model %s", error_check_ret)
201
202
            return error_check_ret

203
204
205
206
207
208
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

209
        return await self.openai_serving_render.render_chat(request)
210
211
212
213
214
215
216
217
218
219
220
221
222

    async def create_chat_completion(
        self,
        request: ChatCompletionRequest,
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
        """
        Chat Completion API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
        Chat Completion API.
        """
223
224
225
226
        # Streaming response
        tokenizer = self.renderer.tokenizer
        assert tokenizer is not None
        reasoning_parser: ReasoningParser | None = None
227
228
229
230
231
232
233
234
235
236
        if self.reasoning_parser_cls:
            # Pass the same chat template kwargs as used in tokenization
            chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                request.chat_template_kwargs,
                self.default_chat_template_kwargs,
            )
            reasoning_parser = self.reasoning_parser_cls(
                tokenizer,
                chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
            )
237
238
239
240
241
        result = await self.render_chat_request(request)
        if isinstance(result, ErrorResponse):
            return result

        conversation, engine_prompts = result
242

243
244
245
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
246
247
248
249
250

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

251
        lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
252

253
        model_name = self.models.model_name(lora_request)
254

255
256
257
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

258
        # Schedule the request and get the result generator.
259
        max_model_len = self.model_config.max_model_len
260
        generators: list[AsyncGenerator[RequestOutput, None]] = []
261
262
263
264
265
266
267
268
        for i, engine_prompt in enumerate(engine_prompts):
            prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids

            # If we are creating sub requests for multiple prompts, ensure that they
            # have unique request ids.
            sub_request_id = (
                request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
            )
269

270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
            max_tokens = get_max_tokens(
                max_model_len,
                request.max_completion_tokens
                if request.max_completion_tokens is not None
                else request.max_tokens,
                self._extract_prompt_len(engine_prompt),
                self.default_sampling_params,
                self.override_max_tokens,
            )

            sampling_params: SamplingParams | BeamSearchParams
            if request.use_beam_search:
                sampling_params = request.to_beam_search_params(
                    max_tokens, self.default_sampling_params
                )
            else:
                sampling_params = request.to_sampling_params(
                    max_tokens,
288
                    self.default_sampling_params,
289
                )
290

291
292
293
294
295
296
            self._log_inputs(
                sub_request_id,
                engine_prompt,
                params=sampling_params,
                lora_request=lora_request,
            )
297

298
299
300
301
302
303
304
305
306
307
            trace_headers = (
                None
                if raw_request is None
                else await self._get_trace_headers(raw_request.headers)
            )

            if isinstance(sampling_params, BeamSearchParams):
                generator = self.beam_search(
                    prompt=engine_prompt,
                    request_id=sub_request_id,
308
309
                    params=sampling_params,
                    lora_request=lora_request,
310
                    trace_headers=trace_headers,
311
                )
312
            else:
313
314
315
316
317
318
319
320
                if not request.include_reasoning:
                    reasoning_ended = True
                elif reasoning_parser:
                    reasoning_ended = reasoning_parser.is_reasoning_end(
                        prompt_token_ids or []
                    )
                else:
                    reasoning_ended = None
321

322
323
324
325
326
327
328
329
330
331
                generator = self.engine_client.generate(
                    engine_prompt,
                    sampling_params,
                    sub_request_id,
                    lora_request=lora_request,
                    trace_headers=trace_headers,
                    priority=request.priority,
                    data_parallel_rank=data_parallel_rank,
                    reasoning_ended=reasoning_ended,
                )
332

333
            generators.append(generator)
334

335
        assert len(generators) == 1
336
        (result_generator,) = generators
337

338
339
        if request.stream:
            return self.chat_completion_stream_generator(
340
341
342
343
344
345
346
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
347
                reasoning_parser,
348
            )
349

350
351
352
353
354
355
356
357
358
359
        return await self.chat_completion_full_generator(
            request,
            result_generator,
            request_id,
            model_name,
            conversation,
            tokenizer,
            request_metadata,
            reasoning_parser,
        )
360
361
362
363

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
364
        return request.messages[-1]["role"]
365

366
    @staticmethod
367
    def _bracket_level(s: str, opening="{", closing="}") -> int:
368
369
370
371
372
373
374
375
376
377
378
379
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
380
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
381
382
383
384
385
386
387
388
389
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
390
            if c == "{":
391
392
                bracket_level += 1
                passed_zero = bracket_level == 0
393
            elif c == "}":
394
395
396
397
398
399
400
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
401
                if c == ",":
402
403
404
405
406
407
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
408
        current_text: str | None,
409
410
        delta_text: str,
        function_name_returned: bool,
411
412
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
413
414
415
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
416
        try:
417
418
419
420
421
422
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
423
            logger.debug("not enough tokens to parse into JSON yet")
424
425
426
427
428
429
430
431
432
433
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
434
435
                delta_text, previous_text
            )
436
437
438
439
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
440
441
442
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
443
444
445
446
447
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
448
449
450
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
451
452
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
453
454
                        arguments, previous_text
                    )
455
456
457
458

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
459
                    if finishes_previous_tool and "parameters" not in current_tool_call:
460
461
462
                        current_tool_call = obj[-2]

                    function_name_returned = True
463
464
465
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
466
467
468
469
470
471
472
473
474
475
476
477
478
479
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
480
481
482

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
483
484
                        delta_text, previous_text
                    )
485
486

                    if delta_text != "":
487
488
489
490
491
492
493
494
495
496
497
498
499
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
500
501
502
503
504
                    else:
                        delta_message = None

        return delta_message, function_name_returned

505
    async def chat_completion_stream_generator(
506
507
508
509
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
510
        model_name: str,
511
        conversation: list[ConversationMessage],
512
        tokenizer: TokenizerLike,
513
        request_metadata: RequestResponseMetadata,
514
        reasoning_parser: ReasoningParser | None = None,
515
    ) -> AsyncGenerator[str, None]:
516
        created_time = int(time.time())
517
        chunk_object_type: Final = "chat.completion.chunk"
518
        first_iteration = True
519
520

        # Send response for each token for each request.n (index)
521
522
523
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
524
        num_prompt_tokens = 0
525
        num_cached_tokens = None
526
527
        if self.use_harmony:
            harmony_parsers = [
528
                get_streamable_parser_for_assistant() for _ in range(num_choices)
529
            ]
530
531
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
532
533
534
535
536
537
538
539
540

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
541
542
            and self._should_stream_with_auto_tool_parsing(request)
        )
543

544
        all_previous_token_ids: list[list[int]] | None
545
        function_name_returned = [False] * num_choices
546
        if self.tool_call_id_type == "kimi_k2":
547
548
549
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
550

551
552
553
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

554
555
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
556
        if tool_choice_auto or reasoning_parser:
557
558
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
559
560
561
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
562
            prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
563
        else:
564
            all_previous_token_ids = None
565

566
567
568
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
569
570
571
572
573
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

574
                tool_parsers: list[ToolParser | None] = [
575
576
577
578
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
579
        except Exception as e:
580
            logger.exception("Error in tool parser creation.")
581
            data = self.create_streaming_error_response(e)
582
583
584
585
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

586
        stream_options = request.stream_options
587
588
589
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
590

591
592
        try:
            async for res in result_generator:
593
594
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
595
596
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
597

598
599
600
601
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
602
                    num_cached_tokens = res.num_cached_tokens
603
604
                    # Send first response for each request.n (index) with
                    # the role
605
                    role = self.get_chat_request_role(request)
606
607
608

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
609
                    for i in range(num_choices):
610
611
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
612
613
614
615
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
616
                            logprobs=None,
617
618
                            finish_reason=None,
                        )
619
620

                        # return prompt_token_ids at the first chunk ever
621
622
623
624
625
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
626
                            model=model_name,
627
628
629
630
631
632
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
633

634
635
636
637
638
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
639
640
                                total_tokens=num_prompt_tokens,
                            )
641

642
643
644
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

645
646
                    # Send response to echo the input portion of the
                    # last message
647
                    if request.echo:
648
                        last_msg_content: str | list[dict[str, str]] = ""
649
650
651
652
653
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
654
                            last_msg_content = conversation[-1]["content"] or ""
655
656

                        if last_msg_content:
657
                            for i in range(num_choices):
658
659
660
661
662
663
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
664
665
666
667
668
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
669
670
                                    model=model_name,
                                )
671
672
673
674
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
675
676
                                        total_tokens=num_prompt_tokens,
                                    )
677

678
                                data = chunk.model_dump_json(exclude_unset=True)
679
680
681
682
683
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
684
                    tool_parser = tool_parsers[i]
685

686
                    if (
687
                        reasoning_parser
688
689
690
691
692
693
694
695
                        and res.prompt_token_ids
                        and prompt_is_reasoning_end_arr[i] is None
                    ):
                        # only check once per choice, because prompt_token_ids
                        # are the same for all deltas in that choice
                        prompt_is_reasoning_end_arr[i] = (
                            reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        )
696
697
698
                    if finish_reason_sent[i]:
                        continue

699
                    if request.logprobs and request.top_logprobs is not None:
700
                        assert output.logprobs is not None, "Did not output logprobs"
701
                        logprobs = self._create_chat_logprobs(
702
703
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
704
                            tokenizer=tokenizer,
705
                            num_output_top_logprobs=request.top_logprobs,
706
                            return_as_token_id=request.return_tokens_as_token_ids,
707
708
709
710
                        )
                    else:
                        logprobs = None

711
712
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
713
                        prev_recipient = harmony_parser.current_recipient
714
715
716

                        # Track accumulated content per token with their state
                        token_states: list[TokenState] = []
717
718
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
719
720
721
722
723
724
725
726
727
                            token_delta = harmony_parser.last_content_delta or ""
                            token_states.append(
                                TokenState(
                                    harmony_parser.current_channel,
                                    harmony_parser.current_recipient,
                                    token_delta,
                                )
                            )
                        delta_text = "".join(delta for _, _, delta in token_states)
728
                        cur_channel = harmony_parser.current_channel
729

730
731
732
733
734
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
735
736
                    else:
                        delta_text = output.text
737

738
739
740
741
742
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
743
744
745
                        # Chunked prefill case, don't return empty chunks
                        continue

746
                    delta_message: DeltaMessage | None
747

748
                    # just update previous_texts and previous_token_ids
749
                    if tool_choice_auto or reasoning_parser:
750
751
752
753
754
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
755
756
                        # avoid the None + list error.
                        if previous_token_ids:
757
                            current_token_ids = previous_token_ids + as_list(
758
759
                                output.token_ids
                            )
760
                        else:
761
                            current_token_ids = as_list(output.token_ids)
762

763
                    if self.use_harmony:
764
765
766
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
767
                                token_states=token_states,
768
769
770
771
772
                                prev_recipient=prev_recipient,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
773
                    # handle streaming deltas for tools with named tool_choice
774
                    elif tool_choice_function_name:
775
776
777
778
779
780
781
782
783
784
785
                        # When encountering think end id in prompt_token_ids
                        # i.e {"enable_thinking": False},
                        # check BEFORE calling the parser to avoid a spurious
                        # reasoning delta on the first chunk.
                        if (
                            reasoning_parser
                            and not reasoning_end_arr[i]
                            and prompt_is_reasoning_end_arr[i]
                        ):
                            reasoning_end_arr[i] = True

786
                        if (
787
                            reasoning_parser
788
789
790
791
792
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
793
794
                            assert reasoning_parser is not None
                            delta_message = (
795
                                reasoning_parser.extract_reasoning_streaming(
796
797
798
799
800
801
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
802
803
                                )
                            )
804
                            # When encountering think end id in delta_token_ids,
805
                            # set reasoning status to end.
806
                            # Only keep 'content', remove 'reasoning'.
807
808
                            if reasoning_parser.is_reasoning_end(
                                as_list(output.token_ids)
809
                            ):
810
                                reasoning_end_arr[i] = True
811
812
813
814
815
816
817
818
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
819
                            if reasoning_parser:
820
821
822
                                delta_text = previous_text + delta_text
                                current_text = ""

823
824
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
825
826
827
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
828
                            else:
829
                                # Generate ID based on tokenizer type
830
                                if is_mistral_tokenizer(tokenizer):
831
832
833
834
835
836
837
                                    tool_call_id = MistralToolCall.generate_random_id()
                                else:
                                    tool_call_id = make_tool_call_id(
                                        id_type=self.tool_call_id_type,
                                        func_name=tool_choice_function_name,
                                        idx=history_tool_call_cnt,
                                    )
838
                                delta_tool_call = DeltaToolCall(
839
                                    id=tool_call_id,
840
841
842
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
843
844
845
846
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
847
                                function_name_returned[i] = True
848
                                history_tool_call_cnt += 1
849

850
851
852
853
854
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
855
                            tools_streamed[i] = True
856

857
858
859
860
861
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
862
863
864
                        output_token_ids = as_list(output.token_ids)

                        if (
865
                            reasoning_parser is not None
866
                            and not reasoning_end_arr[i]
867
                            and prompt_is_reasoning_end_arr[i]
868
869
                        ):
                            reasoning_end_arr[i] = True
870

871
                        if reasoning_parser and not reasoning_end_arr[i]:
872
                            delta_message = (
873
                                reasoning_parser.extract_reasoning_streaming(
874
875
876
877
878
879
880
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
881
                            )
882
883
884
885
886
887
888
889
890
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

891
                        else:
892
                            # either finished reasoning or no reasoning at all
893
                            content = current_text
894
895
896
897
898
899
900
901
902

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
903
                            )
904
905
906
907
908
909
910
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
911

912
913
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
914
                    elif tool_choice_auto and reasoning_parser:
915
916
917
                        assert tool_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
918
                        output_token_ids = as_list(output.token_ids)
919
                        if not reasoning_end_arr[i]:
920
921
922
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
923
                            if prompt_is_reasoning_end_arr[i]:
924
                                reasoning_end_arr[i] = True
925
                                current_token_ids = output_token_ids
926
927
928
929
930
931
932
933
934
935
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
936
937
                                    )
                                )
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
955
956

                        # handle tool calls only after reasoning is done,
957
                        if reasoning_end_arr[i]:
958
                            delta_token_ids = output_token_ids
959
960
961
962
963
964
965
966
967
968
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

969
                            delta_message = tool_parser.extract_tool_calls_streaming(
970
971
                                previous_text=previous_text,
                                current_text=current_text,
972
                                delta_text=delta_text,
973
974
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
992
993
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
994

995
                    # when only reasoning
996
                    elif reasoning_parser:
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
                        # When encountering think end id in prompt_token_ids
                        # i.e {"enable_thinking": False},
                        # set reasoning status to end.
                        # Route all generated tokens as content directly.
                        if prompt_is_reasoning_end_arr[i]:
                            delta_message = DeltaMessage(content=delta_text)
                        else:
                            delta_message = (
                                reasoning_parser.extract_reasoning_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                )
                            )
1014
                    # handle streaming just a content delta
1015
1016
1017
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1018
                    # update the previous values for the next iteration
1019
                    if (tool_choice_auto or reasoning_parser) and not self.use_harmony:
1020
1021
1022
1023
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1024
1025
1026
1027
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1028

1029
                    # set the previous values for the next iteration
1030
                    previous_num_tokens[i] += len(output.token_ids)
1031
1032
1033
1034
1035
1036

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1037
1038
1039
1040
1041
1042
1043
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1044
                            continue
1045
                        delta_message = DeltaMessage()
1046

1047
1048
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
1049
                        delta_content_parts = []
1050
                        if delta_message.content:
1051
                            delta_content_parts.append(delta_message.content)
1052
1053
                        if delta_message.reasoning:
                            reasoning = delta_message.reasoning
1054
1055
1056
                            delta_content_parts.append(f"[reasoning: {reasoning}]")
                        if delta_message.tool_calls:
                            tool_args = "".join(
1057
1058
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1059
1060
                                if tc.function and tc.function.arguments
                            )
1061
1062
                            if tool_args:
                                delta_content_parts.append(f"[tool_calls: {tool_args}]")
1063

1064
1065
                        if delta_content_parts and self.enable_log_deltas:
                            delta_content = " ".join(delta_content_parts)
1066
1067
1068
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1069
                                output_token_ids=as_list(output.token_ids),
1070
1071
1072
1073
1074
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1075
1076
1077
1078
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1079
                            delta=delta_message,
1080
                            logprobs=logprobs,
1081
                            finish_reason=None,
1082
1083
1084
1085
1086
1087
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1088
1089

                    # if the model is finished generating
1090
                    else:
1091
1092
1093
1094
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1095
1096
1097
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1098
                        # only happens if we are NOT using structured outputs
1099
                        auto_tools_called = False
1100
                        if tool_parser:
1101
1102
1103
1104
1105
1106
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1107
1108
1109
                        else:
                            index = 0

1110
1111
1112
1113
1114
1115
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1116
                            latest_delta_len = 0
1117
1118
                            if (
                                isinstance(
1119
                                    delta_message.tool_calls[0].function,
1120
1121
1122
1123
1124
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1125
                                latest_delta_len = len(
1126
1127
                                    delta_message.tool_calls[0].function.arguments
                                )
1128

1129
                            # get the expected call based on partial JSON
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
                            # parsing which "autocompletes" the JSON.
                            # Tool parsers (e.g. Qwen3Coder) store
                            # arguments as a JSON string in
                            # prev_tool_call_arr. Calling json.dumps()
                            # on an already-serialized string would
                            # double-serialize it (e.g. '{"k":1}' becomes
                            # '"{\\"k\\":1}"'), which then causes the
                            # replace() below to fail and append the
                            # entire double-serialized string as a
                            # spurious final delta.
                            args = tool_parser.prev_tool_call_arr[index].get(
                                "arguments", {}
1142
                            )
1143
1144
1145
1146
                            if isinstance(args, str):
                                expected_call = args
                            else:
                                expected_call = json.dumps(args, ensure_ascii=False)
1147

1148
                            # get what we've streamed so far for arguments
1149
                            # for the current tool
1150
1151
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1152
                                actual_call = actual_call[:-latest_delta_len]
1153
1154

                            # check to see if there's anything left to stream
1155
                            remaining_call = expected_call.replace(actual_call, "", 1)
1156
                            # set that as a delta message
1157
1158
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1159
                            )
1160

1161
                        # Send the finish response for each request.n only once
1162
1163
1164
1165
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1166
1167
                        if (
                            auto_tools_called
1168
                            or (tools_streamed[i] and not tool_choice_function_name)
1169
1170
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1171
1172
                            finish_reason_ = "tool_calls"
                        else:
1173
1174
1175
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1176
1177
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1178
                            delta=delta_message,
1179
                            logprobs=logprobs,
1180
                            finish_reason=finish_reason_,
1181
                            stop_reason=output.stop_reason,
1182
1183
1184
1185
1186
1187
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1188

1189
                        finish_reason_sent[i] = True
1190

1191
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1192
1193
1194
1195
1196
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1197
1198
                        model=model_name,
                    )
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1209
                    data = chunk.model_dump_json(exclude_unset=True)
1210
1211
                    yield f"data: {data}\n\n"

1212
1213
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1214
1215
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1216
1217
1218
1219
1220
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1221
1222
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1223
1224
                        cached_tokens=num_cached_tokens
                    )
1225
1226
1227
1228
1229
1230
1231

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1232
1233
1234
1235
1236
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1237
                yield f"data: {final_usage_data}\n\n"
1238

1239
1240
1241
1242
1243
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1244
1245
1246
1247
1248
1249
1250
1251
1252
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1253
1254
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1255
1256
1257
1258
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1259
                        output_token_ids=None,  # Consider also logging all token IDs
1260
1261
1262
1263
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1264

1265
1266
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1267
        except Exception as e:
1268
            logger.exception("Error in chat completion stream generator.")
1269
            data = self.create_streaming_error_response(e)
1270
            yield f"data: {data}\n\n"
1271
1272
1273
1274
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1275
1276
1277
1278
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1279
        model_name: str,
1280
        conversation: list[ConversationMessage],
1281
        tokenizer: TokenizerLike,
1282
        request_metadata: RequestResponseMetadata,
1283
        reasoning_parser: ReasoningParser | None = None,
1284
    ) -> ErrorResponse | ChatCompletionResponse:
1285
1286
        from vllm.tokenizers.mistral import MistralTokenizer

1287
        created_time = int(time.time())
1288
        final_res: RequestOutput | None = None
1289

1290
1291
1292
1293
1294
1295
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")

1296
1297
1298
1299
1300
1301
        if final_res is None:
            return self.create_error_response(
                "No output received from the engine.",
                err_type="InternalServerError",
                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
            )
1302

1303
        choices: list[ChatCompletionResponseChoice] = []
1304
        if self.tool_call_id_type == "kimi_k2":
1305
1306
1307
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1308

1309
1310
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1311
1312
1313
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1314
            token_ids = output.token_ids
1315
            out_logprobs = output.logprobs
1316
            tool_call_info = None
1317

1318
1319
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1320
                logprobs = self._create_chat_logprobs(
1321
                    token_ids=token_ids,
1322
                    top_logprobs=out_logprobs,
1323
                    num_output_top_logprobs=request.top_logprobs,
1324
                    tokenizer=tokenizer,
1325
                    return_as_token_id=request.return_tokens_as_token_ids,
1326
1327
1328
                )
            else:
                logprobs = None
1329
1330

            if self.use_harmony:
1331
                reasoning, content, _ = parse_chat_output(token_ids)
1332
                if not request.include_reasoning:
1333
                    reasoning = None
1334

1335
                if self.tool_parser is not None:
1336
1337
1338
1339
1340
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1341
1342
1343
1344
1345
1346
1347
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1348
                    content = tool_call_info.content
1349
1350
                    message = ChatMessage(
                        role=role,
1351
                        reasoning=reasoning,
1352
1353
1354
1355
1356
1357
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1358
                        reasoning=reasoning,
1359
1360
                        content=content,
                    )
1361
1362
1363
1364
1365

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1366
1367
1368
1369
1370
1371
1372
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1373
                    stop_reason=output.stop_reason,
1374
1375
1376
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1377
1378
1379
                )
                choices.append(choice_data)
                continue
1380

1381
            if reasoning_parser:
1382
1383
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1384
                reasoning, content = reasoning_parser.extract_reasoning(
1385
1386
                    output.text, request=request
                )
1387
                if not request.include_reasoning:
1388
                    reasoning = None
1389
            else:
1390
                reasoning = None
1391
                content = output.text
1392

1393
            auto_tools_called = False
1394
1395
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1396
1397
1398
1399
1400
1401
1402
1403
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
1404
                MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall
1405
            )
1406
            if (not self.enable_auto_tools or not self.tool_parser) and (
1407
1408
1409
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1410
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1411

1412
1413
1414
1415
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1416
                assert tool_calls is not None and len(tool_calls) > 0
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
                tool_call_class_items = []
                for idx, tc in enumerate(tool_calls):
                    # Use native ID if available (e.g., Kimi K2),
                    # otherwise generate ID with correct id_type
                    if tc.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tc.id, function=tc)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(tool_call_class(function=tc))
                        else:
                            generated_id = make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tc.name,
1435
                                idx=history_tool_call_cnt,
1436
1437
1438
1439
1440
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tc)
                            )
                    history_tool_call_cnt += 1
1441
1442
                message = ChatMessage(
                    role=role,
1443
                    reasoning=reasoning,
1444
                    content="",
1445
                    tool_calls=tool_call_class_items,
1446
                )
1447

1448
            elif request.tool_choice and request.tool_choice == "required":
1449
                tool_call_class_items = []
1450
                tool_calls = tool_calls or []
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
                for idx, tool_call in enumerate(tool_calls):
                    # Use native ID if available,
                    # otherwise generate ID with correct id_type
                    if tool_call.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tool_call.id, function=tool_call)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
                        if isinstance(tokenizer, MistralTokenizer):
                            tool_call_class_items.append(
                                tool_call_class(function=tool_call)
                            )
                        else:
                            generated_id = make_tool_call_id(
1468
1469
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
1470
                                idx=history_tool_call_cnt,
1471
1472
1473
1474
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tool_call)
                            )
1475
                    history_tool_call_cnt += 1
1476
1477
1478
                message = ChatMessage(
                    role=role,
                    content="",
1479
                    tool_calls=tool_call_class_items,
1480
                    reasoning=reasoning,
1481
                )
1482

1483
1484
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1485
            elif not request.tool_choice or request.tool_choice == "none":
1486
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1487
1488

            # handle when there are tools and tool choice is auto
1489
1490
1491
1492
1493
1494
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1495
1496
1497
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1498
1499
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
                    tool_call_items = []
                    for idx, tc in enumerate(tool_calls):
                        # Use native ID if available (e.g., Kimi K2),
                        # otherwise generate ID with correct id_type
                        if tc.id:
                            tool_call_items.append(
                                tool_call_class(id=tc.id, function=tc)
                            )
                        else:
                            # Generate ID using the correct format (kimi_k2 or random),
                            # but leave it to the class if it's Mistral to preserve
                            # 9-char IDs
                            if isinstance(tokenizer, MistralTokenizer):
                                tool_call_items.append(tool_call_class(function=tc))
                            else:
                                generated_id = make_tool_call_id(
                                    id_type=self.tool_call_id_type,
                                    func_name=tc.name,
1518
                                    idx=history_tool_call_cnt,
1519
1520
1521
1522
1523
                                )
                                tool_call_items.append(
                                    tool_call_class(id=generated_id, function=tc)
                                )
                        history_tool_call_cnt += 1
1524
1525
                    message = ChatMessage(
                        role=role,
1526
                        reasoning=reasoning,
1527
                        content=content,
1528
                        tool_calls=tool_call_items,
1529
                    )
1530
1531
1532
1533

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1534
1535
1536
1537
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1538
1539
                    if content and len(content) > 0:
                        ret_content = content
1540
1541
                    message = ChatMessage(
                        role=role,
1542
                        reasoning=reasoning,
1543
1544
                        content=ret_content,
                    )
1545
1546
1547
1548
1549
1550

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1551
1552
                    "completion."
                )
1553
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1554
1555
1556
1557
1558
1559
1560
1561
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1562

1563
1564
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1565
                message=message,
1566
                logprobs=logprobs,
1567
1568
1569
1570
1571
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1572
                stop_reason=output.stop_reason,
1573
1574
1575
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1576
            )
1577
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1578

1579
1580
            choices.append(choice_data)

1581
        if request.echo:
1582
            last_msg_content: str | list[dict[str, str]] = ""
1583
1584
1585
1586
1587
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1588
                last_msg_content = conversation[-1]["content"] or ""
1589
            if isinstance(last_msg_content, list):
1590
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1591
1592

            for choice in choices:
1593
                full_message = last_msg_content + (choice.message.content or "")
1594
1595
                choice.message.content = full_message

1596
        assert final_res.prompt_token_ids is not None
1597
        num_prompt_tokens = len(final_res.prompt_token_ids)
1598
1599
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1600
        num_generated_tokens = sum(
1601
1602
1603
1604
1605
1606
1607
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1608
1609
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1610
1611
                cached_tokens=final_res.num_cached_tokens
            )
1612
1613
1614

        request_metadata.final_usage_info = usage

1615
1616
1617
1618
1619
1620
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1621
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1622
1623
1624
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1625
            kv_transfer_params=final_res.kv_transfer_params,
1626
1627
        )

1628
1629
1630
1631
1632
1633
1634
1635
1636
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1637
1638
1639
1640
1641
                    for tc in choice.message.tool_calls:  # type: ignore
                        function_call: FunctionCall = tc.function  # type: ignore
                        tool_call_descriptions.append(
                            f"{function_call.name}({function_call.arguments})"
                        )
1642
1643
1644
1645
1646
1647
1648
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1649
                        output_token_ids = final_res.outputs[choice.index].token_ids
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1660
        return response
1661
1662

    def _get_top_logprobs(
1663
1664
        self,
        logprobs: dict[int, Logprob],
1665
        top_logprobs: int | None,
1666
        tokenizer: TokenizerLike | None,
1667
1668
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1669
        return [
1670
            ChatCompletionLogProb(
1671
1672
1673
1674
1675
1676
1677
1678
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1679
1680
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1681
1682
            )
            for i, p in enumerate(logprobs.items())
1683
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1684
1685
1686
1687
1688
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1689
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1690
        tokenizer: TokenizerLike | None,
1691
1692
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1693
1694
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1695
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1696

1697
1698
1699
1700
1701
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1702
1703
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1704
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1705
                if should_return_as_token_id:
1706
                    token = f"token_id:{token_id}"
1707
                else:
1708
1709
                    if tokenizer is None:
                        raise ValueError(
1710
                            "Unable to get tokenizer because `skip_tokenizer_init=True`"
1711
1712
                        )

1713
                    token = tokenizer.decode(token_id)
1714

1715
1716
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1717
                        token=token,
1718
                        bytes=list(token.encode("utf-8", errors="replace")),
1719
1720
                    )
                )
1721
            else:
1722
1723
1724
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1725
1726
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1727
                        token=self._get_decoded_token(
1728
1729
1730
                            step_token,
                            token_id,
                            tokenizer,
1731
                            should_return_as_token_id,
1732
1733
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1734
1735
1736
1737
1738
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1739
                        top_logprobs=self._get_top_logprobs(
1740
1741
1742
1743
1744
1745
1746
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1747
1748

        return ChatCompletionLogProbs(content=logprobs_content)
1749

1750
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1751
1752
1753
1754
1755
1756
1757
1758
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1759
1760
1761
1762
1763
1764
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1765
1766
1767

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1768
        delta_message: DeltaMessage | None,
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1780
            output.finish_reason is not None
1781
1782
1783
1784
1785
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1786
1787
1788
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1789

1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )