serving.py 79.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from http import HTTPStatus
10
from typing import TYPE_CHECKING, Any, Final
11

12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from partial_json_parser.core.options import Allow
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
22
    get_tool_call_id_type,
23
24
    make_tool_call_id,
)
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.chat_completion.protocol import (
27
28
29
30
31
32
33
34
35
36
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
37
38
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
39
    TokenState,
40
41
42
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
43
44
45
46
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
47
    FunctionCall,
48
49
50
51
52
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
53
from vllm.entrypoints.openai.engine.serving import (
54
55
56
57
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
58
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
59
60
61
62
63
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    parse_chat_output,
)
64
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
65
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
66
from vllm.inputs import EngineInput
67
from vllm.logger import init_logger
68
from vllm.logprobs import Logprob
69
from vllm.outputs import CompletionOutput, RequestOutput
70
from vllm.parser import ParserManager
71
from vllm.parser.abstract_parser import Parser
72
from vllm.reasoning import ReasoningParser
73
from vllm.renderers import ChatParams
74
from vllm.sampling_params import BeamSearchParams, SamplingParams
75
from vllm.tokenizers import TokenizerLike
76
from vllm.tool_parsers.utils import partial_json_loads
77
from vllm.utils.collection_utils import as_list
78
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
79
80
81

if TYPE_CHECKING:
    from vllm.entrypoints.serve.render.serving import OpenAIServingRender
82
83
84
85
86

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
87
88
89
    def __init__(
        self,
        engine_client: EngineClient,
90
        models: OpenAIServingModels,
91
92
        response_role: str,
        *,
93
        openai_serving_render: "OpenAIServingRender",
94
95
        request_logger: RequestLogger | None,
        chat_template: str | None,
96
        chat_template_content_format: ChatTemplateContentFormatOption,
97
        trust_request_chat_template: bool = False,
98
        return_tokens_as_token_ids: bool = False,
99
        reasoning_parser: str = "",
100
        enable_auto_tools: bool = False,
101
        exclude_tools_when_tool_choice_none: bool = False,
102
        tool_parser: str | None = None,
103
        enable_prompt_tokens_details: bool = False,
104
        enable_force_include_usage: bool = False,
105
        enable_log_outputs: bool = False,
106
        enable_log_deltas: bool = True,
107
        default_chat_template_kwargs: dict[str, Any] | None = None,
108
    ) -> None:
109
110
111
112
113
114
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
        )
115

116
        self.openai_serving_render = openai_serving_render
117
        self.response_role = response_role
118
119
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
120
        self.trust_request_chat_template = trust_request_chat_template
121
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
122
        self.enable_log_outputs = enable_log_outputs
123
        self.enable_log_deltas = enable_log_deltas
124

125
        # set up reasoning parser
126
        self.reasoning_parser_cls = ParserManager.get_reasoning_parser(
127
128
            reasoning_parser_name=reasoning_parser
        )
129
130
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
131
132
133
134
        self.tool_parser = ParserManager.get_tool_parser(
            tool_parser_name=tool_parser,
            enable_auto_tools=enable_auto_tools,
            model_name=self.model_config.model,
135
        )
136
137
138
139
140
141
        self.parser_cls = ParserManager.get_parser(
            tool_parser_name=tool_parser,
            reasoning_parser_name=reasoning_parser,
            enable_auto_tools=enable_auto_tools,
            model_name=self.model_config.model,
        )
142
143
144
145
146
147
        if (
            is_mistral_tool_parser(self.tool_parser)
            and self.reasoning_parser_cls is not None
        ):
            from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

148
149
            MistralToolParser.model_can_reason = True

150
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
151

152
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
153
        self.enable_force_include_usage = enable_force_include_usage
154
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
155
156
157
158
159
160
        mc = self.model_config
        self.override_max_tokens = (
            self.default_sampling_params.get("max_tokens")
            if mc.generation_config not in ("auto", "vllm")
            else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
        )
161
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
162
163
164
165
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
166
167
                get_stop_tokens_for_assistant_actions()
            )
168

169
        self.tool_call_id_type = get_tool_call_id_type(self.model_config)
170

171
172
173
174
175
176
177
178
179
180
        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

181
182
183
184
185
186
    def warmup(self) -> None:
        self.renderer.warmup(
            ChatParams(
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                chat_template_kwargs=self.default_chat_template_kwargs,
187
            )
188
        )
189

190
191
192
193
194
195
196
197
198
199
200
201
    def _effective_chat_template_kwargs(
        self, request: ChatCompletionRequest
    ) -> dict[str, Any]:
        return (
            request.build_chat_params(
                self.chat_template,
                self.chat_template_content_format,
            )
            .with_defaults(self.default_chat_template_kwargs)
            .chat_template_kwargs
        )

202
    async def render_chat_request(
203
204
        self,
        request: ChatCompletionRequest,
205
    ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
206
        """
207
208
209
210
        Validate the model and preprocess a chat completion request.

        Delegates preprocessing logic to OpenAIServingRender, adding the
        engine-aware checks (LoRA model validation, engine health).
211

212
        Returns:
213
            A tuple of (conversation, engine_inputs) on success,
214
            or an ErrorResponse on failure.
215
216
217
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
218
            logger.error("Error with model %s", error_check_ret)
219
220
            return error_check_ret

221
222
223
224
225
226
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

227
        return await self.openai_serving_render.render_chat(request)
228
229
230
231
232
233
234
235
236
237
238
239
240

    async def create_chat_completion(
        self,
        request: ChatCompletionRequest,
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
        """
        Chat Completion API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
        Chat Completion API.
        """
241
242
243
        # Streaming response
        tokenizer = self.renderer.tokenizer
        assert tokenizer is not None
244
        chat_template_kwargs = self._effective_chat_template_kwargs(request)
245
        reasoning_parser: ReasoningParser | None = None
246
247
248
249
250
        if self.reasoning_parser_cls:
            reasoning_parser = self.reasoning_parser_cls(
                tokenizer,
                chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
            )
251
252
253
254
        result = await self.render_chat_request(request)
        if isinstance(result, ErrorResponse):
            return result

255
        conversation, engine_inputs = result
256

257
258
259
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
260
261
262
263
264

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

265
        lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
266

267
        model_name = self.models.model_name(lora_request)
268

269
270
271
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

272
        # Schedule the request and get the result generator.
273
        max_model_len = self.model_config.max_model_len
274
        generators: list[AsyncGenerator[RequestOutput, None]] = []
275
276
        for i, engine_input in enumerate(engine_inputs):
            prompt_token_ids = self._extract_prompt_components(engine_input).token_ids
277
278
279
280

            # If we are creating sub requests for multiple prompts, ensure that they
            # have unique request ids.
            sub_request_id = (
281
                request_id if len(engine_inputs) == 1 else f"{request_id}_{i}"
282
            )
283

284
285
286
287
288
            max_tokens = get_max_tokens(
                max_model_len,
                request.max_completion_tokens
                if request.max_completion_tokens is not None
                else request.max_tokens,
289
                self._extract_prompt_len(engine_input),
290
291
292
293
294
295
296
297
298
299
300
301
                self.default_sampling_params,
                self.override_max_tokens,
            )

            sampling_params: SamplingParams | BeamSearchParams
            if request.use_beam_search:
                sampling_params = request.to_beam_search_params(
                    max_tokens, self.default_sampling_params
                )
            else:
                sampling_params = request.to_sampling_params(
                    max_tokens,
302
                    self.default_sampling_params,
303
                )
304

305
306
            self._log_inputs(
                sub_request_id,
307
                engine_input,
308
309
310
                params=sampling_params,
                lora_request=lora_request,
            )
311

312
313
314
315
316
317
318
319
            trace_headers = (
                None
                if raw_request is None
                else await self._get_trace_headers(raw_request.headers)
            )

            if isinstance(sampling_params, BeamSearchParams):
                generator = self.beam_search(
320
                    prompt=engine_input,
321
                    request_id=sub_request_id,
322
323
                    params=sampling_params,
                    lora_request=lora_request,
324
                    trace_headers=trace_headers,
325
                )
326
            else:
327
328
                if not request.include_reasoning:
                    reasoning_ended = True
329
330
331
332
333
                elif request._grammar_from_tool_parser:
                    # The Mistral grammar already includes an optional
                    # `think?` rule that handles both reasoning and
                    # non-reasoning outputs.
                    reasoning_ended = True
334
335
336
337
338
339
                elif reasoning_parser:
                    reasoning_ended = reasoning_parser.is_reasoning_end(
                        prompt_token_ids or []
                    )
                else:
                    reasoning_ended = None
340

341
                generator = self.engine_client.generate(
342
                    engine_input,
343
344
345
346
347
348
349
350
                    sampling_params,
                    sub_request_id,
                    lora_request=lora_request,
                    trace_headers=trace_headers,
                    priority=request.priority,
                    data_parallel_rank=data_parallel_rank,
                    reasoning_ended=reasoning_ended,
                )
351

352
            generators.append(generator)
353

354
        assert len(generators) == 1
355
        (result_generator,) = generators
356

357
358
        if request.stream:
            return self.chat_completion_stream_generator(
359
360
361
362
363
364
365
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
366
                reasoning_parser,
367
                chat_template_kwargs=chat_template_kwargs,
368
            )
369

370
371
372
373
374
375
376
377
378
379
        return await self.chat_completion_full_generator(
            request,
            result_generator,
            request_id,
            model_name,
            conversation,
            tokenizer,
            request_metadata,
            reasoning_parser,
        )
380
381
382
383

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
384
        return request.messages[-1]["role"]
385

386
    @staticmethod
387
    def _bracket_level(s: str, opening="{", closing="}") -> int:
388
389
390
391
392
393
394
395
396
397
398
399
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
400
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
401
402
403
404
405
406
407
408
409
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
410
            if c == "{":
411
412
                bracket_level += 1
                passed_zero = bracket_level == 0
413
            elif c == "}":
414
415
416
417
418
419
420
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
421
                if c == ",":
422
423
424
425
426
427
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
428
        current_text: str | None,
429
430
        delta_text: str,
        function_name_returned: bool,
431
432
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
433
434
435
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
436
        try:
437
438
439
440
441
442
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
443
            logger.debug("not enough tokens to parse into JSON yet")
444
445
446
447
448
449
450
451
452
453
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
454
455
                delta_text, previous_text
            )
456
457
458
459
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
460
461
462
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
463
464
465
466
467
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
468
469
470
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
471
472
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
473
474
                        arguments, previous_text
                    )
475
476
477
478

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
479
                    if finishes_previous_tool and "parameters" not in current_tool_call:
480
481
482
                        current_tool_call = obj[-2]

                    function_name_returned = True
483
484
485
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
486
487
488
489
490
491
492
493
494
495
496
497
498
499
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
500
501
502

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
503
504
                        delta_text, previous_text
                    )
505
506

                    if delta_text != "":
507
508
509
510
511
512
513
514
515
516
517
518
519
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
520
521
522
523
524
                    else:
                        delta_message = None

        return delta_message, function_name_returned

525
    async def chat_completion_stream_generator(
526
527
528
529
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
530
        model_name: str,
531
        conversation: list[ConversationMessage],
532
        tokenizer: TokenizerLike,
533
        request_metadata: RequestResponseMetadata,
534
        reasoning_parser: ReasoningParser | None = None,
535
        chat_template_kwargs: dict[str, Any] | None = None,
536
    ) -> AsyncGenerator[str, None]:
537
        created_time = int(time.time())
538
        chunk_object_type: Final = "chat.completion.chunk"
539
        first_iteration = True
540
541

        # Send response for each token for each request.n (index)
542
543
544
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
545
        num_prompt_tokens = 0
546
        num_cached_tokens = None
547
548
        if self.use_harmony:
            harmony_parsers = [
549
                get_streamable_parser_for_assistant() for _ in range(num_choices)
550
            ]
551
552
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
553

554
555
        is_mistral_grammar_path = request._grammar_from_tool_parser

556
557
558
559
560
561
562
563
        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
564
565
            and self._should_stream_with_auto_tool_parsing(request)
        )
566

567
568
569
570
571
572
573
574
575
576
577
578
579
580
        # Determine whether required/named tool_choice should fall back to
        # the auto tool_parser path instead of the standard JSON-based parsing.
        # This happens when the parser declares supports_required_and_named=False
        # (e.g. GLM models that output XML instead of JSON).
        tool_choice_uses_parser = (
            self.tool_parser is not None
            and not self.tool_parser.supports_required_and_named
            and request.tools
            and (
                request.tool_choice == "required"
                or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
            )
        )

581
        all_previous_token_ids: list[list[int]] | None
582
        function_name_returned = [False] * num_choices
583
        if self.tool_call_id_type == "kimi_k2":
584
585
586
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
587

588
589
590
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

591
592
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
593
594
595
596
597
598
        if (
            is_mistral_grammar_path
            or tool_choice_auto
            or tool_choice_uses_parser
            or reasoning_parser
        ):
599
            # These are only required in "auto" tool choice case
600
            all_previous_token_ids = [[] for _ in range(num_choices)]
601
            reasoning_end_arr = [False] * num_choices
602
            prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
603
        else:
604
            all_previous_token_ids = None
605

606
        try:
607
            if self.parser_cls is not None:
608
609
610
611
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )
612
613
614
615
616
617
                parsers: list[Parser | None] = [
                    self.parser_cls(
                        tokenizer,
                        request.tools,
                        chat_template_kwargs=chat_template_kwargs,
                    )
618
619
                    for _ in range(num_choices)
                ]
620
            else:
621
                parsers = [None] * num_choices
622
        except Exception as e:
623
            logger.exception("Error in parser creation.")
624
            data = self.create_streaming_error_response(e)
625
626
627
628
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

629
        stream_options = request.stream_options
630
631
632
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
633

634
635
        try:
            async for res in result_generator:
636
637
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
638
639
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
640

641
642
643
644
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
645
                    num_cached_tokens = res.num_cached_tokens
646
647
                    # Send first response for each request.n (index) with
                    # the role
648
                    role = self.get_chat_request_role(request)
649
650
651

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
652
                    for i in range(num_choices):
653
654
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
655
656
657
658
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
659
                            logprobs=None,
660
661
                            finish_reason=None,
                        )
662
663

                        # return prompt_token_ids at the first chunk ever
664
665
666
667
668
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
669
                            model=model_name,
670
671
672
673
674
675
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
676

677
678
679
680
681
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
682
683
                                total_tokens=num_prompt_tokens,
                            )
684

685
686
687
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

688
689
                    # Send response to echo the input portion of the
                    # last message
690
                    if request.echo:
691
                        last_msg_content: str | list[dict[str, str]] = ""
692
693
694
695
696
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
697
                            last_msg_content = conversation[-1]["content"] or ""
698
699

                        if last_msg_content:
700
                            for i in range(num_choices):
701
702
703
704
705
706
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
707
708
709
710
711
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
712
713
                                    model=model_name,
                                )
714
715
716
717
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
718
719
                                        total_tokens=num_prompt_tokens,
                                    )
720

721
                                data = chunk.model_dump_json(exclude_unset=True)
722
723
724
725
726
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
727
728
                    parser = parsers[i]
                    tool_parser = parser.tool_parser if parser is not None else None
729

730
                    if (
731
                        reasoning_parser
732
733
734
735
736
737
738
739
                        and res.prompt_token_ids
                        and prompt_is_reasoning_end_arr[i] is None
                    ):
                        # only check once per choice, because prompt_token_ids
                        # are the same for all deltas in that choice
                        prompt_is_reasoning_end_arr[i] = (
                            reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        )
740
741
742
                    if finish_reason_sent[i]:
                        continue

743
                    if request.logprobs and request.top_logprobs is not None:
744
                        assert output.logprobs is not None, "Did not output logprobs"
745
                        logprobs = self._create_chat_logprobs(
746
747
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
748
                            tokenizer=tokenizer,
749
                            num_output_top_logprobs=request.top_logprobs,
750
                            return_as_token_id=request.return_tokens_as_token_ids,
751
752
753
754
                        )
                    else:
                        logprobs = None

755
756
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
757
                        prev_recipient = harmony_parser.current_recipient
758
759
760

                        # Track accumulated content per token with their state
                        token_states: list[TokenState] = []
761
762
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
763
764
765
766
767
768
769
770
771
                            token_delta = harmony_parser.last_content_delta or ""
                            token_states.append(
                                TokenState(
                                    harmony_parser.current_channel,
                                    harmony_parser.current_recipient,
                                    token_delta,
                                )
                            )
                        delta_text = "".join(delta for _, _, delta in token_states)
772
                        cur_channel = harmony_parser.current_channel
773

774
775
776
777
778
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
779
780
                    else:
                        delta_text = output.text
781

782
783
784
785
786
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
787
788
789
                        # Chunked prefill case, don't return empty chunks
                        continue

790
                    delta_message: DeltaMessage | None
791

792
                    # just update previous_texts and previous_token_ids
793
794
795
796
797
798
                    if (
                        is_mistral_grammar_path
                        or tool_choice_auto
                        or tool_choice_uses_parser
                        or reasoning_parser
                    ):
799
800
801
802
803
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
804
805
                        # avoid the None + list error.
                        if previous_token_ids:
806
                            current_token_ids = previous_token_ids + as_list(
807
808
                                output.token_ids
                            )
809
                        else:
810
                            current_token_ids = as_list(output.token_ids)
811

812
                    if self.use_harmony:
813
814
815
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
816
                                token_states=token_states,
817
818
819
820
821
                                prev_recipient=prev_recipient,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
822
823
                    # Mistral grammar path: combined reasoning + tool streaming
                    elif is_mistral_grammar_path:
824
825
826
827
                        from vllm.tool_parsers.mistral_tool_parser import (
                            MistralToolParser,
                        )

828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
                        assert tool_parser is not None
                        assert isinstance(tool_parser, MistralToolParser)
                        assert reasoning_end_arr is not None
                        output_token_ids = as_list(output.token_ids)
                        result = tool_parser.extract_maybe_reasoning_and_tool_streaming(
                            reasoning_parser=reasoning_parser,
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            output_token_ids=output_token_ids,
                            reasoning_ended=reasoning_end_arr[i],
                            prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),
                            request=request,
                        )
                        delta_message = result.delta_message
                        reasoning_end_arr[i] = result.reasoning_ended
                        current_text = result.current_text
                        current_token_ids = result.current_token_ids
                        if result.tools_called:
                            tools_streamed[i] = True
850
                    # handle streaming deltas for tools with named tool_choice
851
852
853
                    # Skip when tool_choice_uses_parser so it falls through
                    # to the auto tool_parser branches below.
                    elif tool_choice_function_name and not tool_choice_uses_parser:
854
855
856
857
858
859
860
861
862
863
864
                        # When encountering think end id in prompt_token_ids
                        # i.e {"enable_thinking": False},
                        # check BEFORE calling the parser to avoid a spurious
                        # reasoning delta on the first chunk.
                        if (
                            reasoning_parser
                            and not reasoning_end_arr[i]
                            and prompt_is_reasoning_end_arr[i]
                        ):
                            reasoning_end_arr[i] = True

865
                        if (
866
                            reasoning_parser
867
868
869
870
871
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
872
873
                            assert reasoning_parser is not None
                            delta_message = (
874
                                reasoning_parser.extract_reasoning_streaming(
875
876
877
878
879
880
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
881
882
                                )
                            )
883
                            # When encountering think end id in delta_token_ids,
884
                            # set reasoning status to end.
885
                            # Only keep 'content', remove 'reasoning'.
886
887
                            if reasoning_parser.is_reasoning_end(
                                as_list(output.token_ids)
888
                            ):
889
                                reasoning_end_arr[i] = True
890
891
892
893
894
895
896
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
897
                            if reasoning_parser:
898
899
900
                                delta_text = previous_text + delta_text
                                current_text = ""

901
902
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
903
904
905
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
906
                            else:
907
                                # Generate ID based on tokenizer type
908
                                if is_mistral_tokenizer(tokenizer):
909
910
911
912
                                    from vllm.tool_parsers.mistral_tool_parser import (
                                        MistralToolCall,
                                    )

913
914
915
916
917
918
919
                                    tool_call_id = MistralToolCall.generate_random_id()
                                else:
                                    tool_call_id = make_tool_call_id(
                                        id_type=self.tool_call_id_type,
                                        func_name=tool_choice_function_name,
                                        idx=history_tool_call_cnt,
                                    )
920
                                delta_tool_call = DeltaToolCall(
921
                                    id=tool_call_id,
922
923
924
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
925
926
927
928
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
929
                                function_name_returned[i] = True
930
                                history_tool_call_cnt += 1
931

932
933
934
935
936
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
937
                            tools_streamed[i] = True
938

939
940
941
942
943
944
                    # Skip when tool_choice_uses_parser so it falls through
                    # to the auto tool_parser branches below.
                    elif (
                        request.tool_choice == "required"
                        and not tool_choice_uses_parser
                    ):
945
946
947
948
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
949
950
951
                        output_token_ids = as_list(output.token_ids)

                        if (
952
                            reasoning_parser is not None
953
                            and not reasoning_end_arr[i]
954
                            and prompt_is_reasoning_end_arr[i]
955
956
                        ):
                            reasoning_end_arr[i] = True
957

958
                        if reasoning_parser and not reasoning_end_arr[i]:
959
                            delta_message = (
960
                                reasoning_parser.extract_reasoning_streaming(
961
962
963
964
965
966
967
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
968
                            )
969
970
971
972
973
974
975
976
977
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

978
                        else:
979
                            # either finished reasoning or no reasoning at all
980
                            content = current_text
981
982
983
984
985
986
987
988
989

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
990
                            )
991
992
993
994
995
996
997
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
998

999
1000
                    elif parser is not None:
                        delta_message = parser.parse_delta(
1001
                            delta_text=delta_text,
1002
                            delta_token_ids=as_list(output.token_ids),
1003
                            request=request,
1004
                            prompt_token_ids=res.prompt_token_ids,
1005
                        )
1006
1007
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1008
                    # handle streaming just a content delta (no parsers)
1009
1010
1011
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1012
                    # update the previous values for the next iteration
1013
                    if (
1014
1015
1016
1017
                        is_mistral_grammar_path
                        or tool_choice_auto
                        or tool_choice_uses_parser
                        or reasoning_parser
1018
                    ) and not self.use_harmony:
1019
1020
1021
1022
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1023
1024
1025
1026
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1027

1028
                    # set the previous values for the next iteration
1029
                    previous_num_tokens[i] += len(output.token_ids)
1030
1031
1032
1033
1034
1035

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1036
1037
1038
1039
1040
1041
1042
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1043
                            continue
1044
                        delta_message = DeltaMessage()
1045

1046
1047
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
1048
                        delta_content_parts = []
1049
                        if delta_message.content:
1050
                            delta_content_parts.append(delta_message.content)
1051
1052
                        if delta_message.reasoning:
                            reasoning = delta_message.reasoning
1053
1054
1055
                            delta_content_parts.append(f"[reasoning: {reasoning}]")
                        if delta_message.tool_calls:
                            tool_args = "".join(
1056
1057
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1058
1059
                                if tc.function and tc.function.arguments
                            )
1060
1061
                            if tool_args:
                                delta_content_parts.append(f"[tool_calls: {tool_args}]")
1062

1063
1064
                        if delta_content_parts and self.enable_log_deltas:
                            delta_content = " ".join(delta_content_parts)
1065
1066
1067
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1068
                                output_token_ids=as_list(output.token_ids),
1069
1070
1071
1072
1073
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1074
1075
1076
1077
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1078
                            delta=delta_message,
1079
                            logprobs=logprobs,
1080
                            finish_reason=None,
1081
1082
1083
1084
1085
1086
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1087
1088

                    # if the model is finished generating
1089
                    else:
1090
1091
1092
1093
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1094
1095
1096
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1097
                        # only happens if we are NOT using structured outputs
1098
                        index = 0
1099
                        auto_tools_called = False
1100
                        if tool_parser:
1101
1102
1103
1104
1105
1106
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1107
                        should_check = (
1108
1109
1110
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
1111
1112
1113
1114
                        )
                        # only check if there are any tool calls
                        # detected by partial parsing
                        if should_check and tool_parser and auto_tools_called:
1115
                            latest_delta_len = 0
1116
1117
                            if (
                                isinstance(
1118
                                    delta_message.tool_calls[0].function,
1119
1120
1121
1122
1123
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1124
                                latest_delta_len = len(
1125
1126
                                    delta_message.tool_calls[0].function.arguments
                                )
1127

1128
                            # get the expected call based on partial JSON
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
                            # parsing which "autocompletes" the JSON.
                            # Tool parsers (e.g. Qwen3Coder) store
                            # arguments as a JSON string in
                            # prev_tool_call_arr. Calling json.dumps()
                            # on an already-serialized string would
                            # double-serialize it (e.g. '{"k":1}' becomes
                            # '"{\\"k\\":1}"'), which then causes the
                            # replace() below to fail and append the
                            # entire double-serialized string as a
                            # spurious final delta.
                            args = tool_parser.prev_tool_call_arr[index].get(
                                "arguments", {}
1141
                            )
1142
1143
1144
1145
                            if isinstance(args, str):
                                expected_call = args
                            else:
                                expected_call = json.dumps(args, ensure_ascii=False)
1146

1147
                            # get what we've streamed so far for arguments
1148
                            # for the current tool
1149
1150
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1151
                                actual_call = actual_call[:-latest_delta_len]
1152
1153

                            # check to see if there's anything left to stream
1154
                            remaining_call = expected_call.replace(actual_call, "", 1)
1155
                            # set that as a delta message
1156
1157
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1158
                            )
1159

1160
                        # Send the finish response for each request.n only once
1161
1162
1163
1164
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1165
1166
                        if (
                            auto_tools_called
1167
                            or (tools_streamed[i] and not tool_choice_function_name)
1168
1169
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1170
1171
                            finish_reason_ = "tool_calls"
                        else:
1172
1173
1174
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1175
1176
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1177
                            delta=delta_message,
1178
                            logprobs=logprobs,
1179
                            finish_reason=finish_reason_,
1180
                            stop_reason=output.stop_reason,
1181
1182
1183
1184
1185
1186
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1187

1188
                        finish_reason_sent[i] = True
1189

1190
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1191
1192
1193
1194
1195
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1196
1197
                        model=model_name,
                    )
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1208
                    data = chunk.model_dump_json(exclude_unset=True)
1209
1210
                    yield f"data: {data}\n\n"

1211
1212
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1213
1214
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1215
1216
1217
1218
1219
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1220
1221
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1222
1223
                        cached_tokens=num_cached_tokens
                    )
1224
1225
1226
1227
1228
1229
1230

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1231
1232
1233
1234
1235
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1236
                yield f"data: {final_usage_data}\n\n"
1237

1238
1239
1240
1241
1242
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1243
1244
1245
1246
1247
1248
1249
1250
1251
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1252
1253
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1254
1255
1256
1257
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1258
                        output_token_ids=None,  # Consider also logging all token IDs
1259
1260
1261
1262
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1263

1264
1265
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1266
        except Exception as e:
1267
            logger.exception("Error in chat completion stream generator.")
1268
            data = self.create_streaming_error_response(e)
1269
            yield f"data: {data}\n\n"
1270
1271
1272
1273
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1274
1275
1276
1277
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1278
        model_name: str,
1279
        conversation: list[ConversationMessage],
1280
        tokenizer: TokenizerLike,
1281
        request_metadata: RequestResponseMetadata,
1282
        reasoning_parser: ReasoningParser | None = None,
1283
    ) -> ErrorResponse | ChatCompletionResponse:
1284
        created_time = int(time.time())
1285
        final_res: RequestOutput | None = None
1286

1287
1288
1289
1290
1291
1292
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")

1293
1294
1295
1296
1297
1298
        if final_res is None:
            return self.create_error_response(
                "No output received from the engine.",
                err_type="InternalServerError",
                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
            )
1299

1300
        choices: list[ChatCompletionResponseChoice] = []
1301
        if self.tool_call_id_type == "kimi_k2":
1302
1303
1304
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1305

1306
1307
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1308
1309
1310
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1311
            token_ids = output.token_ids
1312
            out_logprobs = output.logprobs
1313
            tool_call_info = None
1314

1315
1316
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1317
                logprobs = self._create_chat_logprobs(
1318
                    token_ids=token_ids,
1319
                    top_logprobs=out_logprobs,
1320
                    num_output_top_logprobs=request.top_logprobs,
1321
                    tokenizer=tokenizer,
1322
                    return_as_token_id=request.return_tokens_as_token_ids,
1323
1324
1325
                )
            else:
                logprobs = None
1326
1327

            if self.use_harmony:
1328
                reasoning, content, _ = parse_chat_output(token_ids)
1329
                if not request.include_reasoning:
1330
                    reasoning = None
1331

1332
                if self.tool_parser is not None:
1333
1334
1335
1336
1337
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1338
                    tool_parser = self.tool_parser(tokenizer, request.tools)
1339
1340
1341
1342
1343
1344
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1345
                    content = tool_call_info.content
1346
1347
                    message = ChatMessage(
                        role=role,
1348
                        reasoning=reasoning,
1349
1350
1351
1352
1353
1354
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1355
                        reasoning=reasoning,
1356
1357
                        content=content,
                    )
1358
1359
1360
1361
1362

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1363
1364
1365
1366
1367
1368
1369
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1370
                    stop_reason=output.stop_reason,
1371
1372
1373
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1374
1375
1376
                )
                choices.append(choice_data)
                continue
1377

1378
            if reasoning_parser:
1379
1380
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1381
                reasoning, content = reasoning_parser.extract_reasoning(
1382
1383
                    output.text, request=request
                )
1384
                if not request.include_reasoning:
1385
                    reasoning = None
1386
            else:
1387
                reasoning = None
1388
                content = output.text
1389

1390
            auto_tools_called = False
1391
1392
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1393
1394
1395
1396
1397
1398
1399
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
1400
1401
1402
1403
1404
1405
            if is_mistral_tokenizer(tokenizer):
                from vllm.tool_parsers.mistral_tool_parser import MistralToolCall

                tool_call_class: type[ToolCall] = MistralToolCall
            else:
                tool_call_class = ToolCall
1406
1407
1408

            use_mistral_tool_parser = request._grammar_from_tool_parser
            if use_mistral_tool_parser:
1409
1410
                from vllm.tool_parsers.mistral_tool_parser import MistralToolParser

1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
                tool_call_items = MistralToolParser.build_non_streaming_tool_calls(
                    tool_calls
                )
                if tool_call_items:
                    auto_tools_called = (
                        request.tool_choice is None or request.tool_choice == "auto"
                    )
                message = ChatMessage(
                    role=role,
                    reasoning=reasoning,
                    content=content,
                    tool_calls=tool_call_items,
                )

            elif (not self.enable_auto_tools or not self.tool_parser) and (
1426
1427
1428
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1429
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1430

1431
1432
1433
1434
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1435
                assert tool_calls is not None and len(tool_calls) > 0
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
                tool_call_class_items = []
                for idx, tc in enumerate(tool_calls):
                    # Use native ID if available (e.g., Kimi K2),
                    # otherwise generate ID with correct id_type
                    if tc.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tc.id, function=tc)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
1448
                        if is_mistral_tokenizer(tokenizer):
1449
1450
1451
1452
1453
                            tool_call_class_items.append(tool_call_class(function=tc))
                        else:
                            generated_id = make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tc.name,
1454
                                idx=history_tool_call_cnt,
1455
1456
1457
1458
1459
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tc)
                            )
                    history_tool_call_cnt += 1
1460
1461
                message = ChatMessage(
                    role=role,
1462
                    reasoning=reasoning,
1463
                    content="",
1464
                    tool_calls=tool_call_class_items,
1465
                )
1466

1467
            elif request.tool_choice and request.tool_choice == "required":
1468
                tool_call_class_items = []
1469
                tool_calls = tool_calls or []
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
                for idx, tool_call in enumerate(tool_calls):
                    # Use native ID if available,
                    # otherwise generate ID with correct id_type
                    if tool_call.id:
                        tool_call_class_items.append(
                            tool_call_class(id=tool_call.id, function=tool_call)
                        )
                    else:
                        # Generate ID using the correct format (kimi_k2 or random),
                        # but leave it to the class if it's Mistral to preserve
                        # 9-char IDs
1481
                        if is_mistral_tokenizer(tokenizer):
1482
1483
1484
1485
1486
                            tool_call_class_items.append(
                                tool_call_class(function=tool_call)
                            )
                        else:
                            generated_id = make_tool_call_id(
1487
1488
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
1489
                                idx=history_tool_call_cnt,
1490
1491
1492
1493
                            )
                            tool_call_class_items.append(
                                tool_call_class(id=generated_id, function=tool_call)
                            )
1494
                    history_tool_call_cnt += 1
1495
1496
1497
                message = ChatMessage(
                    role=role,
                    content="",
1498
                    tool_calls=tool_call_class_items,
1499
                    reasoning=reasoning,
1500
                )
1501

1502
1503
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1504
            elif not request.tool_choice or request.tool_choice == "none":
1505
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1506
1507

            # handle when there are tools and tool choice is auto
1508
1509
1510
1511
1512
1513
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1514
1515
1516
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1517
1518
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
                    tool_call_items = []
                    for idx, tc in enumerate(tool_calls):
                        # Use native ID if available (e.g., Kimi K2),
                        # otherwise generate ID with correct id_type
                        if tc.id:
                            tool_call_items.append(
                                tool_call_class(id=tc.id, function=tc)
                            )
                        else:
                            # Generate ID using the correct format (kimi_k2 or random),
                            # but leave it to the class if it's Mistral to preserve
                            # 9-char IDs
1531
                            if is_mistral_tokenizer(tokenizer):
1532
1533
1534
1535
1536
                                tool_call_items.append(tool_call_class(function=tc))
                            else:
                                generated_id = make_tool_call_id(
                                    id_type=self.tool_call_id_type,
                                    func_name=tc.name,
1537
                                    idx=history_tool_call_cnt,
1538
1539
1540
1541
1542
                                )
                                tool_call_items.append(
                                    tool_call_class(id=generated_id, function=tc)
                                )
                        history_tool_call_cnt += 1
1543
1544
                    message = ChatMessage(
                        role=role,
1545
                        reasoning=reasoning,
1546
                        content=content,
1547
                        tool_calls=tool_call_items,
1548
                    )
1549
1550
1551
1552

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1553
1554
1555
1556
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1557
1558
                    if content and len(content) > 0:
                        ret_content = content
1559
1560
                    message = ChatMessage(
                        role=role,
1561
                        reasoning=reasoning,
1562
1563
                        content=ret_content,
                    )
1564
1565
1566
1567
1568
1569

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1570
1571
                    "completion."
                )
1572
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1573
1574
1575
1576
1577
1578
1579
1580
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1581

1582
1583
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1584
                message=message,
1585
                logprobs=logprobs,
1586
1587
1588
1589
1590
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1591
                stop_reason=output.stop_reason,
1592
1593
1594
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1595
            )
1596
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1597

1598
1599
            choices.append(choice_data)

1600
        if request.echo:
1601
            last_msg_content: str | list[dict[str, str]] = ""
1602
1603
1604
1605
1606
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1607
                last_msg_content = conversation[-1]["content"] or ""
1608
            if isinstance(last_msg_content, list):
1609
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1610
1611

            for choice in choices:
1612
                full_message = last_msg_content + (choice.message.content or "")
1613
1614
                choice.message.content = full_message

1615
        assert final_res.prompt_token_ids is not None
1616
        num_prompt_tokens = len(final_res.prompt_token_ids)
1617
1618
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1619
        num_generated_tokens = sum(
1620
1621
1622
1623
1624
1625
1626
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1627
1628
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1629
1630
                cached_tokens=final_res.num_cached_tokens
            )
1631
1632
1633

        request_metadata.final_usage_info = usage

1634
1635
1636
1637
1638
1639
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1640
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1641
1642
1643
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1644
            kv_transfer_params=final_res.kv_transfer_params,
1645
1646
        )

1647
1648
1649
1650
1651
1652
1653
1654
1655
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1656
1657
1658
1659
1660
                    for tc in choice.message.tool_calls:  # type: ignore
                        function_call: FunctionCall = tc.function  # type: ignore
                        tool_call_descriptions.append(
                            f"{function_call.name}({function_call.arguments})"
                        )
1661
1662
1663
1664
1665
1666
1667
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1668
                        output_token_ids = final_res.outputs[choice.index].token_ids
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1679
        return response
1680
1681

    def _get_top_logprobs(
1682
1683
        self,
        logprobs: dict[int, Logprob],
1684
        top_logprobs: int | None,
1685
        tokenizer: TokenizerLike | None,
1686
1687
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1688
        return [
1689
            ChatCompletionLogProb(
1690
1691
1692
1693
1694
1695
1696
1697
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1698
1699
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1700
1701
            )
            for i, p in enumerate(logprobs.items())
1702
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1703
1704
1705
1706
1707
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1708
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1709
        tokenizer: TokenizerLike | None,
1710
1711
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1712
1713
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1714
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1715

1716
1717
1718
1719
1720
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1721
1722
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1723
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1724
                if should_return_as_token_id:
1725
                    token = f"token_id:{token_id}"
1726
                else:
1727
1728
                    if tokenizer is None:
                        raise ValueError(
1729
                            "Unable to get tokenizer because `skip_tokenizer_init=True`"
1730
1731
                        )

1732
                    token = tokenizer.decode(token_id)
1733

1734
1735
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1736
                        token=token,
1737
                        bytes=list(token.encode("utf-8", errors="replace")),
1738
1739
                    )
                )
1740
            else:
1741
1742
1743
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1744
1745
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1746
                        token=self._get_decoded_token(
1747
1748
1749
                            step_token,
                            token_id,
                            tokenizer,
1750
                            should_return_as_token_id,
1751
1752
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1753
1754
1755
1756
1757
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1758
                        top_logprobs=self._get_top_logprobs(
1759
1760
1761
1762
1763
1764
1765
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1766
1767

        return ChatCompletionLogProbs(content=logprobs_content)
1768

1769
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1770
1771
1772
1773
1774
1775
1776
1777
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1778
1779
1780
1781
1782
1783
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1784
1785
1786

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1787
        delta_message: DeltaMessage | None,
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1799
            output.finish_reason is not None
1800
1801
1802
1803
1804
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1805
1806
1807
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1808

1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )